diff --git a/src/Analyser/ResultCache/CollectedDataSpool.php b/src/Analyser/ResultCache/CollectedDataSpool.php new file mode 100644 index 00000000000..839ee9b2451 --- /dev/null +++ b/src/Analyser/ResultCache/CollectedDataSpool.php @@ -0,0 +1,301 @@ + */ + private int $size = 0; + + private int|false $creatorPid; + + /** @var array>, array, int<1, max>, string}>>> collector type => file => [offset, length, hash] of each value list */ + private array $segments = []; + + /** @var array file => when its first value list came, to keep the order of the files the same as in memory */ + private array $fileOrder = []; + + /** @var array collector type => order of the last file it got */ + private array $lastFileOrder = []; + + /** @var array collector types that got a file out of order, because the file came in an earlier job too */ + private array $unordered = []; + + public function __construct(private string $path) + { + $this->creatorPid = getmypid(); + } + + /** + * @param CollectorData $collectedData + */ + public function append(array $collectedData): void + { + if ($collectedData === []) { + return; + } + + $handle = $this->open($this->created ? 'a' : 'w'); + $this->created = true; + $size = $this->size; + $segments = []; + try { + foreach ($collectedData as $file => $dataPerCollector) { + foreach ($dataPerCollector as $collectorType => $values) { + $blob = serialize($values); + $segments[] = [$collectorType, $file, $this->write($handle, $blob), md5($blob)]; + } + } + } catch (CouldNotWriteFileException $e) { + // what was written of this batch is not indexed, so the caller can keep the whole batch elsewhere + @ftruncate($handle, $size); + $this->size = $size; + + throw $e; + } finally { + fclose($handle); + } + + foreach ($segments as [$collectorType, $file, [$offset, $length], $hash]) { + $order = $this->fileOrder[$file] ??= count($this->fileOrder); + if (!isset($this->segments[$collectorType][$file])) { + if (($this->lastFileOrder[$collectorType] ?? -1) > $order) { + $this->unordered[$collectorType] = true; + } else { + $this->lastFileOrder[$collectorType] = $order; + } + } + $this->segments[$collectorType][$file][] = [$offset, $length, $hash]; + } + } + + /** + * Everything appended so far, in memory. For when the spool cannot be written any more. + * + * @return CollectorData + */ + public function toArray(): array + { + $data = []; + if ($this->segments === []) { + return $data; + } + + $handle = $this->openForReading(); + try { + foreach ($this->segments as $collectorType => $segmentsPerFile) { + foreach ($segmentsPerFile as $file => $segments) { + foreach ($segments as [$offset, $length]) { + foreach ($this->read($handle, $file, $offset, $length) as $value) { + $data[$file][$collectorType][] = $value; + } + } + } + } + } finally { + fclose($handle); + } + + return $data; + } + + /** + * Merges the value lists of a file that came in more than one job. The data of a trait is stored + * under the trait's file, and each file that uses the trait can be in a different job. + */ + public function finish(): ?SpooledCollectedData + { + if ($this->segments === []) { + return null; + } + + $entries = []; + $readHandle = null; + $writeHandle = null; + $fileOrder = $this->fileOrder; + try { + foreach ($this->segments as $collectorType => $segmentsPerFile) { + if (isset($this->unordered[$collectorType])) { + uksort($segmentsPerFile, static fn (string $a, string $b): int => $fileOrder[$a] <=> $fileOrder[$b]); + } + foreach ($segmentsPerFile as $file => $segments) { + if (count($segments) === 1) { + $entries[$collectorType][$file] = $segments[0]; + continue; + } + + $readHandle ??= $this->openForReading(); + $merged = []; + foreach ($segments as [$offset, $length]) { + foreach ($this->read($readHandle, $file, $offset, $length) as $value) { + $merged[] = $value; + } + } + + $writeHandle ??= $this->open('a'); + $blob = serialize($merged); + [$offset, $length] = $this->write($writeHandle, $blob); + $entries[$collectorType][$file] = [$offset, $length, md5($blob)]; + } + } + } finally { + if ($readHandle !== null) { + fclose($readHandle); + } + if ($writeHandle !== null) { + fclose($writeHandle); + } + } + + $files = []; + foreach (array_keys($fileOrder) as $file) { + $files[$file] = true; + } + + $this->segments = []; + $this->fileOrder = []; + $this->lastFileOrder = []; + $this->unordered = []; + + return new SpooledCollectedData($this, $entries, $files); + } + + /** + * @return resource + */ + public function openForReading() + { + return $this->open('r'); + } + + /** + * @param resource $handle from openForReading() + * @param int<0, max> $offset + * @param int<1, max> $length + * @return list + */ + public function read($handle, string $file, int $offset, int $length): array + { + return $this->unserializeValues($this->readBlob($handle, $file, $offset, $length), $file); + } + + /** + * The serialized value list, as append() wrote it. + * + * @param resource $handle from openForReading() + * @param int<0, max> $offset + * @param int<1, max> $length + */ + public function readBlob($handle, string $file, int $offset, int $length): string + { + if (fseek($handle, $offset) !== 0) { + throw new RuntimeException(sprintf('Cannot seek to the collected data of %s in %s.', $file, $this->path)); + } + + $blob = fread($handle, $length); + if ($blob === false || strlen($blob) !== $length) { + throw new RuntimeException(sprintf('Expected %d bytes of the collected data of %s in %s.', $length, $file, $this->path)); + } + + return $blob; + } + + /** + * @return list + */ + public function unserializeValues(string $blob, string $file): array + { + $values = @unserialize($blob); + if (!is_array($values) || !array_is_list($values)) { + throw new RuntimeException(sprintf('The collected data of %s in %s is not a list.', $file, $this->path)); + } + + return $values; + } + + public function __destruct() + { + // a forked worker holds a copy of this object and must not delete the file of the main process + if (!$this->created || getmypid() !== $this->creatorPid) { + return; + } + + @unlink($this->path); + } + + /** + * @param resource $handle + * @return array{int<0, max>, int<1, max>} + */ + private function write($handle, string $blob): array + { + $length = strlen($blob); + if ($length === 0 || @fwrite($handle, $blob) !== $length) { + $error = error_get_last(); + throw new CouldNotWriteFileException($this->path, $error !== null ? $error['message'] : 'unknown cause'); + } + + $offset = $this->size; + $this->size += $length; + + return [$offset, $length]; + } + + /** + * Opened for each call: a handle kept open would be inherited by the workers forked later. + * + * @return resource + */ + private function open(string $mode) + { + $handle = @fopen($this->path, $mode); + if ($handle === false) { + $error = error_get_last(); + if ($mode === 'r') { + throw new RuntimeException(sprintf('Cannot open %s: %s', $this->path, $error !== null ? $error['message'] : 'unknown cause')); + } + + throw new CouldNotWriteFileException($this->path, $error !== null ? $error['message'] : 'unknown cause'); + } + + return $handle; + } + +} diff --git a/src/Analyser/ResultCache/LazyCollectedData.php b/src/Analyser/ResultCache/LazyCollectedData.php index b51cd3c6db7..aec8f424500 100644 --- a/src/Analyser/ResultCache/LazyCollectedData.php +++ b/src/Analyser/ResultCache/LazyCollectedData.php @@ -7,6 +7,7 @@ use PHPStan\ShouldNotHappenException; use function array_diff_key; use function array_key_exists; +use function array_keys; use function count; use function ksort; use function md5; @@ -18,7 +19,8 @@ * * The cached part is an index of where each analysed file's entry sits in the cache file; the * fresh part is what this run's analysis produced. Until something reads it, the main process - * holds a few bytes per file, which is all the forked workers inherit. + * holds a few bytes per file, which is all the forked workers inherit. A parallel analysis writes + * its fresh data to a CollectedDataSpool (the spooled part); other analyses keep it in memory. * * iterate() reads the data of one collector, one file at a time. toArray() reads all of it. * @@ -39,8 +41,13 @@ public function __construct( private array $cachedIndex, private ?CollectedDataReader $cachedReader, private array $fresh, + private ?SpooledCollectedData $spooled = null, ) { + if ($spooled !== null && $spooled->isEmpty()) { + $this->spooled = null; + } + if ($cachedIndex !== [] && $cachedReader === null) { throw new ShouldNotHappenException(); } @@ -71,7 +78,7 @@ public function withRenamedFile(string $from, string $to): self $fresh[$file === $from ? $to : $file] = $data; } - return new self($this->cachedIndex, $this->cachedReader, $fresh); + return new self($this->cachedIndex, $this->cachedReader, $fresh, $this->spooled !== null ? $this->spooled->withRenamedFile($from, $to) : null); } /** @@ -90,15 +97,20 @@ public function getFresh(): array return $this->fresh; } + public function getSpooled(): ?SpooledCollectedData + { + return $this->spooled; + } + public function isEmpty(): bool { - return count($this->cachedIndex) === 0 && count($this->fresh) === 0; + return count($this->cachedIndex) === 0 && count($this->fresh) === 0 && $this->spooled === null; } /** * The values of one collector, one analysed file at a time: the cached files in the order of the - * cache file, then the fresh ones. A fresh file replaces its cached entry and comes with the fresh - * ones. Holds only the file being yielded in memory. + * cache file, then the spooled ones, then the fresh ones. A spooled or fresh file replaces its + * cached entry. Holds only the file being yielded in memory. * * @return Generator> */ @@ -115,6 +127,10 @@ public function iterate(string $collectorType): Generator } } + if ($this->spooled !== null) { + yield from $this->spooled->iterate($collectorType); + } + foreach ($this->fresh as $file => $data) { if (!array_key_exists($collectorType, $data)) { continue; @@ -133,7 +149,11 @@ public function hashCollector(string $collectorType): string { $hashes = []; if ($this->cachedIndex !== []) { - $hashes = array_diff_key($this->getCollectorLocations()->getHashes($collectorType), $this->fresh); + $hashes = $this->withoutSpooledAndFresh($this->getCollectorLocations()->getHashes($collectorType)); + } + + if ($this->spooled !== null) { + $hashes = $this->spooled->getHashes($collectorType) + $hashes; } foreach ($this->fresh as $file => $data) { @@ -167,16 +187,21 @@ public function getUnreadableException(): ?UnreadableCollectedDataException */ public function toArray(): array { - if (count($this->cachedIndex) === 0 || $this->cachedReader === null) { - return $this->fresh; - } + $result = []; + if (count($this->cachedIndex) !== 0 && $this->cachedReader !== null) { + try { + $result = $this->cachedReader->read($this->cachedIndex); + } catch (UnreadableCollectedDataException $e) { + $this->unreadableException = $e; - try { - $result = $this->cachedReader->read($this->cachedIndex); - } catch (UnreadableCollectedDataException $e) { - $this->unreadableException = $e; + throw $e; + } + } - throw $e; + if ($this->spooled !== null) { + foreach (array_keys($this->spooled->getFiles()) as $file) { + $result[$file] = $this->spooled->read($file); + } } foreach ($this->fresh as $file => $data) { @@ -195,7 +220,22 @@ private function getCachedPositions(string $collectorType): array return []; } - return array_diff_key($this->getCollectorLocations()->getPositions($collectorType), $this->fresh); + return $this->withoutSpooledAndFresh($this->getCollectorLocations()->getPositions($collectorType)); + } + + /** + * @template T + * @param array $cached + * @return array + */ + private function withoutSpooledAndFresh(array $cached): array + { + $cached = array_diff_key($cached, $this->fresh); + if ($this->spooled !== null) { + $cached = array_diff_key($cached, $this->spooled->getFiles()); + } + + return $cached; } /** diff --git a/src/Analyser/ResultCache/ResultCacheManager.php b/src/Analyser/ResultCache/ResultCacheManager.php index 48b27837989..a2930454a2d 100644 --- a/src/Analyser/ResultCache/ResultCacheManager.php +++ b/src/Analyser/ResultCache/ResultCacheManager.php @@ -1450,6 +1450,7 @@ public function process(AnalyserResult $analyserResult, ResultCache $resultCache && $linesToIgnore === $resultCache->getLinesToIgnore() && $unmatchedLineIgnores === $resultCache->getUnmatchedLineIgnores() && $collectedData->getFresh() === [] + && $collectedData->getSpooled() === null && $collectedData->getCachedIndex() === $resultCache->getCollectedData()->getCachedIndex() && $collectedDataRuleResults === $resultCache->getCollectedDataRuleResults() && $filesWithCollectedDataRuleErrors === $resultCache->getFilesWithCollectedDataRuleErrors() @@ -1512,7 +1513,7 @@ public function process(AnalyserResult $analyserResult, ResultCache $resultCache $errorsByFile = $this->mergeErrors($resultCache, $freshErrorsByFile); $locallyIgnoredErrorsByFile = $this->mergeLocallyIgnoredErrors($resultCache, $freshLocallyIgnoredErrorsByFile); - $collectedData = $this->mergeCollectedData($resultCache, $freshCollectedData->getFresh()); + $collectedData = $this->mergeCollectedData($resultCache, $freshCollectedData); $dependencies = $this->mergeDependencies($resultCache->getDependencies(), $resultCache->getFilesToAnalyse(), $analyserResult->getDependencies()); $usedTraitDependencies = $this->mergeDependencies($resultCache->getUsedTraitDependencies(), $resultCache->getFilesToAnalyse(), $analyserResult->getUsedTraitDependencies()); $valueDependencies = $this->mergeValueDependencies($resultCache->getValueDependencies(), $resultCache->getFilesToAnalyse(), $analyserResult->getValueDependencies()); @@ -1630,26 +1631,28 @@ private function mergeLocallyIgnoredErrors(ResultCache $resultCache, array $fres return $errorsByFile; } - /** - * @param CollectorData $freshCollectedDataByFile - */ - private function mergeCollectedData(ResultCache $resultCache, array $freshCollectedDataByFile): LazyCollectedData + private function mergeCollectedData(ResultCache $resultCache, LazyCollectedData $freshCollectedData): LazyCollectedData { $cachedIndex = $resultCache->getCollectedData()->getCachedIndex(); + $freshCollectedDataByFile = $freshCollectedData->getFresh(); $mergedFresh = []; + $analysedFiles = []; foreach ($resultCache->getFilesToAnalyse() as $file) { if (array_key_exists($file, $this->fileReplacements)) { unset($cachedIndex[$file]); $file = $this->fileReplacements[$file]; } unset($cachedIndex[$file]); + $analysedFiles[$file] = true; if (!array_key_exists($file, $freshCollectedDataByFile)) { continue; } $mergedFresh[$file] = $freshCollectedDataByFile[$file]; } - return new LazyCollectedData($cachedIndex, $this->createCollectedDataReader(), $mergedFresh); + $spooled = $freshCollectedData->getSpooled(); + + return new LazyCollectedData($cachedIndex, $this->createCollectedDataReader(), $mergedFresh, $spooled !== null ? $spooled->only($analysedFiles) : null); } /** @@ -2178,13 +2181,14 @@ private function save( } } - return new LazyCollectedData($savedCollectedDataIndex, $this->createCollectedDataReader(), $collectedData->getFresh()); + return new LazyCollectedData($savedCollectedDataIndex, $this->createCollectedDataReader(), $collectedData->getFresh(), $collectedData->getSpooled()); } /** * The collected data section: the cached entries are copied byte for byte from the file they - * were restored from, the fresh ones are serialized. Written in the order of the file paths, as - * every other section is, so the file does not depend on which files were re-analysed. + * were restored from, the spooled and fresh ones are serialized, one file at a time. Written in + * the order of the file paths, as every other section is, so the file does not depend on which + * files were re-analysed. * * @param resource $handle * @return array Where every copied entry is in the new file @@ -2194,13 +2198,22 @@ private function writeCollectedDataFrame($handle, string $file, LazyCollectedDat $transformer = $this->getPathTransformer(); $cachedIndex = $collectedData->getCachedIndex(); $fresh = $collectedData->getFresh(); - $files = array_keys($cachedIndex + $fresh); + $spooled = $collectedData->getSpooled(); + $files = array_keys($cachedIndex + $fresh + ($spooled !== null ? $spooled->getFiles() : [])); sort($files); $this->writeToHandle($handle, $file, 'collectedData* ' . count($files) . "\n"); $sourceHandle = null; + $spoolHandle = null; $savedIndex = []; foreach ($files as $analysedFile) { + if (!array_key_exists($analysedFile, $fresh) && $spooled !== null && array_key_exists($analysedFile, $spooled->getFiles())) { + $spoolHandle ??= $spooled->openForReading(); + $this->writeSpooledCollectedDataEntry($handle, $file, $analysedFile, $spooled, $spoolHandle); + + continue; + } + if (array_key_exists($analysedFile, $fresh)) { $collectedDataPerFile = $fresh[$analysedFile]; ksort($collectedDataPerFile); @@ -2233,6 +2246,10 @@ private function writeCollectedDataFrame($handle, string $file, LazyCollectedDat fclose($sourceHandle); } + if ($spoolHandle !== null) { + fclose($spoolHandle); + } + return $savedIndex; } @@ -2252,11 +2269,49 @@ private function writeCollectedDataFrame($handle, string $file, LazyCollectedDat private function writeCollectedDataEntry($handle, string $file, string $relativeFile, array $dataPerCollector, array $dataInMemory): void { $transformer = $this->getPathTransformer(); - $header = []; - $blobs = []; + $blobsAndHashes = []; foreach ($dataPerCollector as $collectorType => $values) { $blob = serialize($values); $hash = $transformer->hasPathsInCollectedData($collectorType) ? md5(serialize($dataInMemory[$collectorType])) : md5($blob); + $blobsAndHashes[$collectorType] = [$blob, $hash]; + } + + $this->writeCollectedDataEntryFrame($handle, $file, $relativeFile, $blobsAndHashes); + } + + /** + * The same entry as writeCollectedDataEntry() writes. The spool holds each value list serialized + * with absolute paths and hashed, so only the values of a collector with paths are serialized again. + * + * @param resource $handle + * @param resource $spoolHandle + */ + private function writeSpooledCollectedDataEntry($handle, string $file, string $analysedFile, SpooledCollectedData $spooled, $spoolHandle): void + { + $transformer = $this->getPathTransformer(); + $blobsAndHashes = $spooled->readBlobs($analysedFile, $spoolHandle); + ksort($blobsAndHashes); + foreach ($blobsAndHashes as $collectorType => [$blob, $hash]) { + if (!$transformer->hasPathsInCollectedData($collectorType)) { + continue; + } + + $values = $transformer->relativizeCollectorValues($collectorType, $spooled->unserializeValues($blob, $analysedFile)); + $blobsAndHashes[$collectorType] = [serialize($values), $hash]; + } + + $this->writeCollectedDataEntryFrame($handle, $file, $transformer->relativizePath($analysedFile), $blobsAndHashes); + } + + /** + * @param resource $handle + * @param array $blobsAndHashes collector type => [serialized values, hash] + */ + private function writeCollectedDataEntryFrame($handle, string $file, string $relativeFile, array $blobsAndHashes): void + { + $header = []; + $blobs = []; + foreach ($blobsAndHashes as $collectorType => [$blob, $hash]) { $header[] = $collectorType . ' ' . strlen($blob) . ' ' . $hash; $blobs[] = $blob; } diff --git a/src/Analyser/ResultCache/ResultCachePathTransformer.php b/src/Analyser/ResultCache/ResultCachePathTransformer.php index defddc82a76..a356a209346 100644 --- a/src/Analyser/ResultCache/ResultCachePathTransformer.php +++ b/src/Analyser/ResultCache/ResultCachePathTransformer.php @@ -186,6 +186,17 @@ private function transformCollectedData(array $collectedData, callable $transfor return $result; } + /** + * The values one collector collected in one file, with the paths in them made relative. + * + * @param list $collectedValues + * @return list + */ + public function relativizeCollectorValues(string $collectorType, array $collectedValues): array + { + return $this->transformCollectorValues($collectorType, $collectedValues, fn (string $path): string => $this->relativizeCompoundKey($path)); + } + /** * The values one collector collected in one file, with the paths in them made absolute again. * diff --git a/src/Analyser/ResultCache/SpooledCollectedData.php b/src/Analyser/ResultCache/SpooledCollectedData.php new file mode 100644 index 00000000000..cde704d0789 --- /dev/null +++ b/src/Analyser/ResultCache/SpooledCollectedData.php @@ -0,0 +1,180 @@ +>, array, int<1, max>, string}>> $entries collector type => file => [offset, length, hash] + * @param array $files every file in $entries, in the order they came in + */ + public function __construct( + private CollectedDataSpool $spool, + private array $entries, + private array $files, + ) + { + } + + /** + * @return array + */ + public function getFiles(): array + { + return $this->files; + } + + public function isEmpty(): bool + { + return $this->entries === []; + } + + /** + * @param array $files + */ + public function only(array $files): self + { + $entries = []; + foreach ($this->entries as $collectorType => $entriesPerFile) { + $kept = array_intersect_key($entriesPerFile, $files); + if ($kept === []) { + continue; + } + + $entries[$collectorType] = $kept; + } + + return new self($this->spool, $entries, array_intersect_key($this->files, $files)); + } + + public function withRenamedFile(string $from, string $to): self + { + $entries = []; + foreach ($this->entries as $collectorType => $entriesPerFile) { + foreach ($entriesPerFile as $file => $entry) { + $entries[$collectorType][$file === $from ? $to : $file] = $entry; + } + } + + $files = []; + foreach (array_keys($this->files) as $file) { + $files[$file === $from ? $to : $file] = true; + } + + return new self($this->spool, $entries, $files); + } + + /** + * @return Generator> + */ + public function iterate(string $collectorType): Generator + { + $entries = $this->entries[$collectorType] ?? []; + if ($entries === []) { + return; + } + + $handle = $this->spool->openForReading(); + try { + foreach ($entries as $file => [$offset, $length]) { + yield $file => $this->spool->read($handle, $file, $offset, $length); + } + } finally { + // also when the caller stops iterating early + fclose($handle); + } + } + + /** + * @return array file => hash + */ + public function getHashes(string $collectorType): array + { + $hashes = []; + foreach ($this->entries[$collectorType] ?? [] as $file => [, , $hash]) { + $hashes[$file] = $hash; + } + + return $hashes; + } + + /** + * @return resource + */ + public function openForReading() + { + return $this->spool->openForReading(); + } + + /** + * The serialized value lists of one file with their hashes, without unserializing them. + * + * @param resource $handle from openForReading() + * @return array>, array{string, string}> collector type => [serialized values, hash] + */ + public function readBlobs(string $file, $handle): array + { + $blobs = []; + foreach ($this->entries as $collectorType => $entriesPerFile) { + if (!array_key_exists($file, $entriesPerFile)) { + continue; + } + + [$offset, $length, $hash] = $entriesPerFile[$file]; + $blobs[$collectorType] = [$this->spool->readBlob($handle, $file, $offset, $length), $hash]; + } + + return $blobs; + } + + /** + * @return list + */ + public function unserializeValues(string $blob, string $file): array + { + return $this->spool->unserializeValues($blob, $file); + } + + /** + * @param resource|null $handle from openForReading(), to read many files without opening the spool for each + * @return array>, list> collector type => values + */ + public function read(string $file, $handle = null): array + { + $data = []; + $ownHandle = null; + try { + foreach ($this->entries as $collectorType => $entriesPerFile) { + if (!array_key_exists($file, $entriesPerFile)) { + continue; + } + + if ($handle === null) { + $handle = $ownHandle = $this->spool->openForReading(); + } + [$offset, $length] = $entriesPerFile[$file]; + $data[$collectorType] = $this->spool->read($handle, $file, $offset, $length); + } + } finally { + if ($ownHandle !== null) { + fclose($ownHandle); + } + } + + return $data; + } + +} diff --git a/src/Parallel/ParallelAnalyser.php b/src/Parallel/ParallelAnalyser.php index 894a9bf5bfb..dc4d8e5c105 100644 --- a/src/Parallel/ParallelAnalyser.php +++ b/src/Parallel/ParallelAnalyser.php @@ -9,15 +9,18 @@ use PHPStan\Analyser\AnalyserResult; use PHPStan\Analyser\Error; use PHPStan\Analyser\InternalError; +use PHPStan\Analyser\ResultCache\CollectedDataSpool; use PHPStan\Analyser\ResultCache\LazyCollectedData; use PHPStan\Analyser\ValueDependencyCollector; use PHPStan\Cache\ArenaCache; +use PHPStan\Collectors\CollectedData; use PHPStan\Command\CommandHelper; use PHPStan\Command\Output; use PHPStan\Dependency\ExportedNodeDecoder; use PHPStan\Dependency\RootExportedNode; use PHPStan\DependencyInjection\AutowiredParameter; use PHPStan\DependencyInjection\AutowiredService; +use PHPStan\File\CouldNotWriteFileException; use PHPStan\Process\ProcessHelper; use PHPStan\Process\TerminationSignal; use PHPStan\Reflection\BetterReflection\SourceLocator\PreForkDirectorySymbolScanner; @@ -35,6 +38,7 @@ use function defined; use function escapeshellarg; use function getenv; +use function getmypid; use function ini_get; use function max; use function parse_url; @@ -42,6 +46,9 @@ use function str_contains; use const PHP_URL_PORT; +/** + * @phpstan-import-type CollectorData from CollectedData + */ #[AutowiredService] final class ParallelAnalyser { @@ -52,6 +59,8 @@ final class ParallelAnalyser private ProcessPool $processPool; + private static int $spoolCount = 0; + public function __construct( #[AutowiredParameter] private int $internalErrorsCountLimit, @@ -59,6 +68,8 @@ public function __construct( float $processTimeout, #[AutowiredParameter(ref: '%parallel.buffer%')] private int $decoderBufferSize, + #[AutowiredParameter] + private string $tmpDir, private ForkParallelChecker $forkParallelChecker, private PreForkDirectorySymbolScanner $preForkDirectorySymbolScanner, private WorkerRunner $workerRunner, @@ -122,6 +133,13 @@ public function analyse( $internalErrors = []; $internalErrorsCount = 0; $collectedData = []; + // Editor mode renames the temporary file in the collected data later, and it analyses few + // files, so it keeps them in memory. + // Named after the process like the temporary file of the result cache, so that the file of a run + // that was killed is reused rather than left behind with the others. The counter keeps apart the + // analyses of a process that runs more than one. + $pid = getmypid(); + $spool = $tmpFile === null ? new CollectedDataSpool(sprintf('%s/collectedData.%s-%d.spool', $this->tmpDir, $pid === false ? Random::generate() : $pid, self::$spoolCount++)) : null; $dependencies = []; $usedTraitDependencies = []; $valueDependencies = ['values' => [], 'dependents' => []]; @@ -137,7 +155,7 @@ public function analyse( $useFork = $this->forkParallelChecker->isSupported(); $server = new TcpServer('127.0.0.1:0', $loop); - $this->processPool = new ProcessPool($server, static function () use ($deferred, &$jobs, &$internalErrors, &$internalErrorsCount, &$reachedInternalErrorsCountLimit, &$errors, &$filteredPhpErrors, &$allPhpErrors, &$locallyIgnoredErrors, &$linesToIgnore, &$unmatchedLineIgnores, &$collectedData, &$dependencies, &$usedTraitDependencies, &$valueDependencies, &$packageDependencies, &$exportedNodes, &$peakMemoryUsages, &$allProcessedFiles, $arenaName): void { + $this->processPool = new ProcessPool($server, static function () use ($deferred, &$jobs, &$internalErrors, &$internalErrorsCount, &$reachedInternalErrorsCountLimit, &$errors, &$filteredPhpErrors, &$allPhpErrors, &$locallyIgnoredErrors, &$linesToIgnore, &$unmatchedLineIgnores, &$collectedData, &$spool, &$dependencies, &$usedTraitDependencies, &$valueDependencies, &$packageDependencies, &$exportedNodes, &$peakMemoryUsages, &$allProcessedFiles, $arenaName): void { if ($arenaName !== null) { ArenaCache::destroy(); } @@ -161,7 +179,7 @@ public function analyse( linesToIgnore: $linesToIgnore, unmatchedLineIgnores: $unmatchedLineIgnores, internalErrors: $internalErrors, - collectedData: LazyCollectedData::fromArray($collectedData), + collectedData: self::finishCollectedData($collectedData, $spool), dependencies: $internalErrorsCount === 0 ? $dependencies : null, usedTraitDependencies: $internalErrorsCount === 0 ? $usedTraitDependencies : null, valueDependencies: $internalErrorsCount === 0 ? $valueDependencies : null, @@ -288,7 +306,7 @@ public function analyse( $insteadOfFile, $input, ); - $process->start(function (array $json) use ($process, &$internalErrors, &$errors, &$filteredPhpErrors, &$allPhpErrors, &$locallyIgnoredErrors, &$linesToIgnore, &$unmatchedLineIgnores, &$collectedData, &$dependencies, &$usedTraitDependencies, &$valueDependencies, &$packageDependencies, &$exportedNodes, &$peakMemoryUsages, &$jobs, $postFileCallback, &$internalErrorsCount, &$reachedInternalErrorsCountLimit, $processIdentifier, $onFileAnalysisHandler, &$allProcessedFiles): void { + $process->start(function (array $json) use ($process, &$internalErrors, &$errors, &$filteredPhpErrors, &$allPhpErrors, &$locallyIgnoredErrors, &$linesToIgnore, &$unmatchedLineIgnores, &$collectedData, &$spool, &$dependencies, &$usedTraitDependencies, &$valueDependencies, &$packageDependencies, &$exportedNodes, &$peakMemoryUsages, &$jobs, $postFileCallback, &$internalErrorsCount, &$reachedInternalErrorsCountLimit, $processIdentifier, $onFileAnalysisHandler, &$allProcessedFiles): void { $fileErrors = []; foreach ($json['errors'] as $jsonError) { $fileErrors[] = Error::decode($jsonError); @@ -322,6 +340,17 @@ public function analyse( $locallyIgnoredErrors[] = $locallyIgnoredFileError; } + if ($spool !== null) { + try { + $spool->append($json['collectedData']); + $json['collectedData'] = []; + } catch (CouldNotWriteFileException) { + // the disk is full or the tmpDir is not writable: keep all of it in memory, as without the spool + $collectedData = $spool->toArray(); + $spool = null; + } + } + foreach ($json['collectedData'] as $file => $jsonDataByCollector) { foreach ($jsonDataByCollector as $collectorType => $listOfCollectedData) { foreach ($listOfCollectedData as $rawCollectedData) { @@ -482,6 +511,22 @@ public function analyse( return $deferred->promise(); } + /** + * @param CollectorData $collectedData + */ + private static function finishCollectedData(array $collectedData, ?CollectedDataSpool $spool): LazyCollectedData + { + if ($spool === null) { + return LazyCollectedData::fromArray($collectedData); + } + + try { + return new LazyCollectedData([], null, $collectedData, $spool->finish()); + } catch (CouldNotWriteFileException) { + return LazyCollectedData::fromArray($spool->toArray()); + } + } + /** * @param string[] $allAnalysedFiles * @param string[] $commandOptions diff --git a/tests/PHPStan/Analyser/ResultCache/CollectedDataSpoolTest.php b/tests/PHPStan/Analyser/ResultCache/CollectedDataSpoolTest.php new file mode 100644 index 00000000000..637b2ebb980 --- /dev/null +++ b/tests/PHPStan/Analyser/ResultCache/CollectedDataSpoolTest.php @@ -0,0 +1,171 @@ +directory = sys_get_temp_dir() . '/phpstan-spool-' . uniqid(); + mkdir($this->directory); + } + + #[Override] + protected function tearDown(): void + { + @unlink($this->path()); + @rmdir($this->directory); + } + + public function testNothingAppendedCreatesNoFile(): void + { + $spool = new CollectedDataSpool($this->path()); + $spool->append([]); + + $this->assertNull($spool->finish()); + $this->assertSame([], $spool->toArray()); + $this->assertFalse(is_file($this->path())); + } + + public function testSpooledDataIsTheSameAsInMemory(): void + { + $batches = [ + [ + 'a.php' => [self::METHOD => ['a1', 'a2'], self::FUNC_CALL => [['call', 1]]], + 'trait.php' => [self::METHOD => ['from a']], + ], + [ + 'b.php' => [self::FUNC_CALL => [['call', 2]]], + // a trait is stored under its own file, and each file using it can be in a different job + 'trait.php' => [self::METHOD => ['from b'], self::FUNC_CALL => [['call', 3]]], + ], + ]; + $inMemory = []; + $spool = new CollectedDataSpool($this->path()); + foreach ($batches as $batch) { + $spool->append($batch); + foreach ($batch as $file => $dataPerCollector) { + foreach ($dataPerCollector as $collectorType => $values) { + foreach ($values as $value) { + $inMemory[$file][$collectorType][] = $value; + } + } + } + } + + $this->assertSame($this->sorted($inMemory), $this->sorted($spool->toArray())); + + $spooled = $spool->finish(); + $this->assertNotNull($spooled); + $this->assertSame(['a.php' => true, 'trait.php' => true, 'b.php' => true], $spooled->getFiles()); + $this->assertSame($inMemory['trait.php'], $spooled->read('trait.php')); + + $lazy = new LazyCollectedData([], null, [], $spooled); + $fromArray = LazyCollectedData::fromArray($inMemory); + foreach ([self::METHOD, self::FUNC_CALL] as $collectorType) { + $this->assertSame(iterator_to_array($fromArray->iterate($collectorType)), iterator_to_array($lazy->iterate($collectorType))); + $this->assertSame($fromArray->hashCollector($collectorType), $lazy->hashCollector($collectorType)); + } + $this->assertSame(md5(serialize(['from a', 'from b'])), $spooled->getHashes(self::METHOD)['trait.php']); + $this->assertSame($inMemory, $lazy->toArray()); + } + + public function testSpooledFilesReplaceCachedOnes(): void + { + $spool = new CollectedDataSpool($this->path()); + $spool->append(['b.php' => [self::METHOD => ['spooled b']], 'c.php' => [self::METHOD => ['spooled c']]]); + $spooled = $spool->finish(); + + $reader = new FakeCollectedDataReader(['a.php' => [self::METHOD => ['cached a']], 'b.php' => [self::METHOD => ['cached b']]]); + $lazy = new LazyCollectedData(['a.php' => [0, 1], 'b.php' => [1, 1]], $reader, [], $spooled); + + $this->assertSame([ + 'a.php' => ['cached a'], + 'b.php' => ['spooled b'], + 'c.php' => ['spooled c'], + ], iterator_to_array($lazy->iterate(self::METHOD))); + $this->assertSame([ + 'a.php' => [self::METHOD => ['cached a']], + 'b.php' => [self::METHOD => ['spooled b']], + 'c.php' => [self::METHOD => ['spooled c']], + ], $lazy->toArray()); + $this->assertSame( + LazyCollectedData::fromArray(['a.php' => [self::METHOD => ['cached a']], 'b.php' => [self::METHOD => ['spooled b']], 'c.php' => [self::METHOD => ['spooled c']]])->hashCollector(self::METHOD), + $lazy->hashCollector(self::METHOD), + ); + } + + public function testOnlyAndRename(): void + { + $spool = new CollectedDataSpool($this->path()); + $spool->append(['a.php' => [self::METHOD => ['a']], 'tmp.php' => [self::METHOD => ['tmp'], self::FUNC_CALL => ['call']]]); + $spooled = $spool->finish(); + $this->assertNotNull($spooled); + + $this->assertSame(['a.php' => ['a']], iterator_to_array($spooled->only(['a.php' => true])->iterate(self::METHOD))); + $this->assertTrue($spooled->only(['other.php' => true])->isEmpty()); + + $renamed = (new LazyCollectedData([], null, [], $spooled))->withRenamedFile('tmp.php', 'real.php'); + $this->assertSame(['a.php' => ['a'], 'real.php' => ['tmp']], iterator_to_array($renamed->iterate(self::METHOD))); + $this->assertSame(['real.php' => ['call']], iterator_to_array($renamed->iterate(self::FUNC_CALL))); + } + + public function testFileIsDeletedWithTheSpool(): void + { + $spool = new CollectedDataSpool($this->path()); + $spool->append(['a.php' => [self::METHOD => ['a']]]); + $spooled = $spool->finish(); + $this->assertTrue(is_file($this->path())); + + unset($spool); + $this->assertTrue(is_file($this->path()), 'the spooled data still reads the file'); + + unset($spooled); + gc_collect_cycles(); + $this->assertFalse(is_file($this->path())); + } + + private function path(): string + { + return $this->directory . '/collectedData.spool'; + } + + /** + * @param array>> $data + * @return array>> + */ + private function sorted(array $data): array + { + ksort($data); + foreach ($data as $file => $dataPerCollector) { + ksort($dataPerCollector); + $data[$file] = $dataPerCollector; + } + + return $data; + } + +} diff --git a/tests/PHPStan/Analyser/ResultCache/ResultCacheManagerCollectedDataTest.php b/tests/PHPStan/Analyser/ResultCache/ResultCacheManagerCollectedDataTest.php index 42e741345d6..316283669f5 100644 --- a/tests/PHPStan/Analyser/ResultCache/ResultCacheManagerCollectedDataTest.php +++ b/tests/PHPStan/Analyser/ResultCache/ResultCacheManagerCollectedDataTest.php @@ -154,6 +154,59 @@ public function testPartialSaveEqualsFreshSaveOfMergedData(): void $this->assertSame($partial, $this->cacheContentsWithoutTime()); } + public function testSpooledFullSaveEqualsFreshSave(): void + { + $data = $this->collectedData(['b', 'a', 'c']); + $data[$this->files[0]][PossiblyPureFuncCallCollector::class] = ['call in a']; + // a collector with paths in its values is stored with them relative + $data[$this->files[2]][ConstantConditionInTraitCollector::class] = [['SomeRule', 'SomeTrait', 'key', true, new Error('An error.', $this->files[2], 3)]]; + $this->saveFullAnalysis($data); + $fromArrays = $this->cacheContentsWithoutTime(); + + @unlink($this->cacheFilePath()); + $manager = $this->createManager(); + $resultCache = $manager->restore($this->files, false, false, null, $this->nullOutput()); + $this->assertTrue($resultCache->isFullAnalysis()); + $spooled = $this->spooled($data); + $processed = $manager->process($this->analyserResult($spooled), $resultCache, $this->nullOutput(), false, true); + $this->assertTrue($processed->save(new FinalizerResult($processed->getAnalyserResult(), [], []))); + + $this->assertSame($fromArrays, $this->cacheContentsWithoutTime()); + $this->assertStringNotContainsString($this->directory, $fromArrays); + $this->assertEquals($data, $this->savedCollectedData($processed)); + } + + public function testSpooledPartialSaveEqualsFreshSave(): void + { + $this->waitUntilTheFilesAreInThePast(); + $this->saveFullAnalysis($this->collectedData(['a', 'b', 'c'])); + + file_put_contents($this->files[1], "waitUntilTheFilesAreInThePast(); + $manager = $this->createManager(); + $resultCache = $manager->restore($this->files, false, false, null, $this->nullOutput()); + $this->assertSame([$this->files[1]], $resultCache->getFilesToAnalyse()); + + // the data of a file that is not re-analysed is left out, as it is from the data in memory + $spooled = $this->spooled([ + $this->files[1] => [MethodWithoutImpurePointsCollector::class => ['fresh b']], + $this->files[2] => [MethodWithoutImpurePointsCollector::class => ['not re-analysed c']], + ]); + $processed = $manager->process($this->analyserResult($spooled), $resultCache, $this->nullOutput(), false, true); + $mergedSpooled = $processed->getAnalyserResult()->getLazyCollectedData()->getSpooled(); + $this->assertNotNull($mergedSpooled); + $this->assertSame([$this->files[1] => true], $mergedSpooled->getFiles()); + $this->assertTrue($processed->save(new FinalizerResult($processed->getAnalyserResult(), [], []))); + + $expected = $this->collectedData(['a', 'c', 'b']); + $expected[$this->files[1]] = [MethodWithoutImpurePointsCollector::class => ['fresh b']]; + $this->assertSame($expected, $this->savedCollectedData($processed)); + + $partial = $this->cacheContentsWithoutTime(); + $this->saveFullAnalysis($expected); + $this->assertSame($partial, $this->cacheContentsWithoutTime()); + } + public function testDamagedValueDiscardsTheCacheWhenRead(): void { $this->saveFullAnalysis($this->collectedData(['a', 'b', 'c'])); @@ -423,12 +476,12 @@ private function saveFullAnalysis(array $collectedData): void } /** - * @param CollectorData $collectedData + * @param CollectorData|LazyCollectedData $collectedData * @param array> $dependencies of the files that depend on others * @param list $filesWithExportedNodes analysed files whose exported nodes are read now * @param list $filesWithErrors */ - private function analyserResult(array $collectedData, array $dependencies = [], array $filesWithExportedNodes = [], array $filesWithErrors = []): AnalyserResult + private function analyserResult(array|LazyCollectedData $collectedData, array $dependencies = [], array $filesWithExportedNodes = [], array $filesWithErrors = []): AnalyserResult { foreach ($this->files as $file) { $dependencies[$file] ??= []; @@ -453,7 +506,7 @@ private function analyserResult(array $collectedData, array $dependencies = [], linesToIgnore: [], unmatchedLineIgnores: [], internalErrors: [], - collectedData: LazyCollectedData::fromArray($collectedData), + collectedData: $collectedData instanceof LazyCollectedData ? $collectedData : LazyCollectedData::fromArray($collectedData), dependencies: $dependencies, usedTraitDependencies: [], valueDependencies: ['values' => [], 'dependents' => []], @@ -532,6 +585,17 @@ private function savedCollectedData(ResultCacheProcessResult $processed): array return $collectedData->toArray(); } + /** + * @param CollectorData $collectedData + */ + private function spooled(array $collectedData): LazyCollectedData + { + $spool = new CollectedDataSpool($this->directory . '/collectedData.spool'); + $spool->append($collectedData); + + return new LazyCollectedData([], null, [], $spool->finish()); + } + private function cacheFilePath(): string { return $this->directory . '/resultCache.php';