From 664e9b016423b21f2c62f6346d10179808c8f234 Mon Sep 17 00:00:00 2001 From: Abdul Malik Ikhsan Date: Sun, 6 Sep 2026 12:58:43 +0700 Subject: [PATCH 1/2] perf: Split cache hydration across parallel workers --- src/Analyser/Analyser.php | 77 ++-------------- src/Analyser/AnalysisNodeExtractor.php | 88 ++++++++++++++++--- src/Analyser/ExtractionResult.php | 14 +++ src/Analyser/Parallel/AnalysisNodeWorker.php | 2 +- .../ParallelAnalysisNodeExtractor.php | 55 ++++++++---- .../Parallel/WorkerProgressHandler.php | 19 +++- tests/Analyser/AnalysisNodeExtractorTest.php | 53 +++++++++++ .../ParallelAnalysisNodeExtractorTest.php | 70 +++++++++++++++ .../Parallel/WorkerProgressHandlerTest.php | 11 ++- 9 files changed, 284 insertions(+), 105 deletions(-) diff --git a/src/Analyser/Analyser.php b/src/Analyser/Analyser.php index d5e73326..a6092291 100644 --- a/src/Analyser/Analyser.php +++ b/src/Analyser/Analyser.php @@ -36,10 +36,8 @@ use function array_key_exists; use function array_keys; use function array_merge; -use function array_push; use function array_unique; use function array_values; -use function count; use function getcwd; use function in_array; use function is_dir; @@ -1290,63 +1288,13 @@ private function collectAnalysisNodes( ?AnalyserOptions $analyserOptions = null, bool $withFileAnalysis = true, ): ExtractionResult { - $classNodes = []; - $fileAnalyses = []; - $anonymousClassNodes = []; - $fileReferences = []; - $fileInstantiations = []; - $functionNodes = []; - $anonymousFunctionNodes = []; - $filesToParse = []; - - foreach ($files as $file) { - $cachedResult = $withFileAnalysis - ? $this->analysisResultCache?->loadAnalysisNodesWithFileAnalysis( - $file, - $this->analysisNodeCacheNamespace - ) - : $this->analysisResultCache?->loadAnalysisNodes($file, $this->analysisNodeCacheNamespace); - - if ($cachedResult === null) { - $filesToParse[] = $file; - continue; - } - - array_push($classNodes, ...$cachedResult['classNodes']); - array_push($anonymousClassNodes, ...$cachedResult['anonymousClassNodes']); - array_push($functionNodes, ...$cachedResult['functionNodes']); - array_push($anonymousFunctionNodes, ...$cachedResult['anonymousFunctionNodes']); - - $fileReferences[$file] = $cachedResult['fileReferences']; - $fileInstantiations[$file] = $cachedResult['fileInstantiations']; - - if (isset($cachedResult['fileAnalysis'])) { - $fileAnalyses[$file] = $cachedResult['fileAnalysis']; - } - } - - $progressHandler?->start(count($filesToParse)); - - if ($filesToParse === []) { - $progressHandler?->finish(); - - return new ExtractionResult( - $classNodes, - $fileAnalyses, - $anonymousClassNodes, - $fileReferences, - $fileInstantiations, - $functionNodes, - $anonymousFunctionNodes, - ); - } - $options = $analyserOptions ?? AnalyserOptions::parallel(); + // Each extractor hydrates its files' node-cache payloads itself, so in + // parallel mode that work is split across the workers instead of being + // done serially here before any worker starts. if ($options->isParallel()) { - // Workers write their own files' cache payloads while other workers are - // still parsing, instead of the coordinator doing it serially afterwards. - $parsedResult = (new ParallelAnalysisNodeExtractor( + $extractionResult = (new ParallelAnalysisNodeExtractor( $this->basePath, $layers, $layerPatterns, @@ -1354,27 +1302,18 @@ private function collectAnalysisNodes( $this->analysisResultCache?->getCacheDirectory(), $this->analysisResultCache, $this->analysisNodeCacheNamespace, - ))->extract($filesToParse, $progressHandler, $withFileAnalysis); + ))->extract($files, $progressHandler, $withFileAnalysis); } else { - $parsedResult = (new AnalysisNodeExtractor( + $extractionResult = (new AnalysisNodeExtractor( $chainLayerResolver, analysisResultCache: $this->analysisResultCache, analysisNodeCacheNamespace: $this->analysisNodeCacheNamespace, - ))->extract($filesToParse, $progressHandler, $withFileAnalysis); + ))->extract($files, $progressHandler, $withFileAnalysis); } $progressHandler?->finish(); - // Cached nodes first, then the freshly parsed ones. - return new ExtractionResult( - classNodes: [...$classNodes, ...$parsedResult->classNodes], - fileAnalyses: $fileAnalyses + $parsedResult->fileAnalyses, - anonymousClassNodes: [...$anonymousClassNodes, ...$parsedResult->anonymousClassNodes], - fileReferences: $fileReferences + $parsedResult->fileReferences, - fileInstantiations: $fileInstantiations + $parsedResult->fileInstantiations, - functionNodes: [...$functionNodes, ...$parsedResult->functionNodes], - anonymousFunctionNodes: [...$anonymousFunctionNodes, ...$parsedResult->anonymousFunctionNodes], - ); + return $extractionResult; } /** diff --git a/src/Analyser/AnalysisNodeExtractor.php b/src/Analyser/AnalysisNodeExtractor.php index b0974f77..3f4eac8c 100644 --- a/src/Analyser/AnalysisNodeExtractor.php +++ b/src/Analyser/AnalysisNodeExtractor.php @@ -11,6 +11,9 @@ use PhpParser\NodeTraverser; use PhpParser\NodeVisitor\NameResolver; +use function array_push; +use function count; + /** * @internal */ @@ -34,24 +37,33 @@ public function __construct( $this->fileAnalysisProvider = $fileAnalysisProvider ?? new FileAnalysisProvider(); } - /** @param list $files */ + /** + * Only files without a valid node-cache payload are parsed, and only those + * count towards the progress total. + * + * @param list $files + */ public function extract( array $files, ?ProgressHandlerInterface $progressHandler = null, bool $withFileAnalysis = true, ): ExtractionResult { + [$cachedResult, $filesToParse] = $this->loadFromCache($files, $withFileAnalysis); + + $progressHandler?->start(count($filesToParse)); + $analysisNodeCollector = new AnalysisNodeCollector($this->layerResolver); $nodeTraverser = new NodeTraverser(new NameResolver(), $analysisNodeCollector); $fileAnalyses = []; - foreach ($files as $file) { + foreach ($filesToParse as $fileToParse) { try { - $ast = $this->fileAnalysisProvider->ast($file, $withFileAnalysis); + $ast = $this->fileAnalysisProvider->ast($fileToParse, $withFileAnalysis); $nonCanonicalKeywordConstants = []; $numericLiterals = []; if ($ast !== null && $ast !== []) { - $analysisNodeCollector->setCurrentFile($file, $this->fileAnalysisProvider->tokens()); + $analysisNodeCollector->setCurrentFile($fileToParse, $this->fileAnalysisProvider->tokens()); $nodeTraverser->traverse($ast); $nonCanonicalKeywordConstants = $analysisNodeCollector->getNonCanonicalKeywordConstants(); @@ -61,18 +73,18 @@ public function extract( // Analysed after the traversal so the facts only the collector // records reach the file analysis without a second AST walk. if ($withFileAnalysis) { - $fileAnalyses[$file] = $this->fileAnalysisProvider->analyse( - $file, + $fileAnalyses[$fileToParse] = $this->fileAnalysisProvider->analyse( + $fileToParse, $nonCanonicalKeywordConstants, $numericLiterals, ); } } finally { if ($withFileAnalysis) { - $this->fileAnalysisProvider->releaseAst($file); + $this->fileAnalysisProvider->releaseAst($fileToParse); } - $progressHandler?->advance($file); + $progressHandler?->advance($fileToParse); } } @@ -87,11 +99,67 @@ public function extract( ); $this->analysisResultCache?->storeExtractionResult( - $files, + $filesToParse, $this->analysisNodeCacheNamespace, $extractionResult ); - return $extractionResult; + return $cachedResult->merge($extractionResult); + } + + /** + * Hydrates every file with a valid node-cache payload; the rest still need parsing. + * + * @param list $files + * @return array{ExtractionResult, list} + */ + private function loadFromCache(array $files, bool $withFileAnalysis): array + { + $classNodes = []; + $fileAnalyses = []; + $anonymousClassNodes = []; + $fileReferences = []; + $fileInstantiations = []; + $functionNodes = []; + $anonymousFunctionNodes = []; + $filesToParse = []; + + foreach ($files as $file) { + $cachedResult = $withFileAnalysis + ? $this->analysisResultCache?->loadAnalysisNodesWithFileAnalysis( + $file, + $this->analysisNodeCacheNamespace + ) + : $this->analysisResultCache?->loadAnalysisNodes($file, $this->analysisNodeCacheNamespace); + + if ($cachedResult === null) { + $filesToParse[] = $file; + continue; + } + + array_push($classNodes, ...$cachedResult['classNodes']); + array_push($anonymousClassNodes, ...$cachedResult['anonymousClassNodes']); + array_push($functionNodes, ...$cachedResult['functionNodes']); + array_push($anonymousFunctionNodes, ...$cachedResult['anonymousFunctionNodes']); + + $fileReferences[$file] = $cachedResult['fileReferences']; + $fileInstantiations[$file] = $cachedResult['fileInstantiations']; + + if (isset($cachedResult['fileAnalysis'])) { + $fileAnalyses[$file] = $cachedResult['fileAnalysis']; + } + } + + $cachedResult = new ExtractionResult( + $classNodes, + $fileAnalyses, + $anonymousClassNodes, + $fileReferences, + $fileInstantiations, + $functionNodes, + $anonymousFunctionNodes, + ); + + return [$cachedResult, $filesToParse]; } } diff --git a/src/Analyser/ExtractionResult.php b/src/Analyser/ExtractionResult.php index 94efb6c0..37bddce9 100644 --- a/src/Analyser/ExtractionResult.php +++ b/src/Analyser/ExtractionResult.php @@ -27,4 +27,18 @@ public function __construct( public array $anonymousFunctionNodes = [], ) { } + + /** Nodes of this result first, then those of $other. */ + public function merge(self $other): self + { + return new self( + [...$this->classNodes, ...$other->classNodes], + $this->fileAnalyses + $other->fileAnalyses, + [...$this->anonymousClassNodes, ...$other->anonymousClassNodes], + $this->fileReferences + $other->fileReferences, + $this->fileInstantiations + $other->fileInstantiations, + [...$this->functionNodes, ...$other->functionNodes], + [...$this->anonymousFunctionNodes, ...$other->anonymousFunctionNodes], + ); + } } diff --git a/src/Analyser/Parallel/AnalysisNodeWorker.php b/src/Analyser/Parallel/AnalysisNodeWorker.php index bfdf45ac..e68addcf 100644 --- a/src/Analyser/Parallel/AnalysisNodeWorker.php +++ b/src/Analyser/Parallel/AnalysisNodeWorker.php @@ -55,7 +55,7 @@ public static function run(string $inputFile, string $outputFile, mixed $outputS $stream = $outputStream ?? STDOUT; - $progressHandler = $emitProgress ? new WorkerProgressHandler($stream) : null; + $progressHandler = $emitProgress ? new WorkerProgressHandler($stream, $files) : null; $cache = $payload['cache'] ?? null; /** @var string $cacheNamespace */ diff --git a/src/Analyser/Parallel/ParallelAnalysisNodeExtractor.php b/src/Analyser/Parallel/ParallelAnalysisNodeExtractor.php index a6c4e457..668b488a 100644 --- a/src/Analyser/Parallel/ParallelAnalysisNodeExtractor.php +++ b/src/Analyser/Parallel/ParallelAnalysisNodeExtractor.php @@ -18,14 +18,17 @@ use function array_fill; use function array_key_exists; use function array_keys; +use function array_pop; use function array_push; use function array_search; use function arsort; use function assert; use function count; use function dirname; +use function explode; use function fclose; use function feof; +use function fgets; use function file_put_contents; use function filesize; use function fread; @@ -39,7 +42,6 @@ use function serialize; use function sprintf; use function stream_set_blocking; -use function substr_count; use function unlink; use function unserialize; use function usleep; @@ -77,6 +79,8 @@ public function extract( bool $withFileAnalysis = true, ): ExtractionResult { if ($files === []) { + $progressHandler?->start(0); + return new ExtractionResult([], []); } @@ -131,20 +135,33 @@ public function extract( assert(isset($pipes[0]) && isset($pipes[1])); fclose($pipes[0]); - $stdoutPipe = $pipes[1]; - stream_set_blocking($stdoutPipe, false); - $pending[] = [ - 'process' => $process, - 'files' => $chunk, - 'filesAdvanced' => 0, - 'inputFile' => $inputFile, - 'outputFile' => $outputFile, - 'stderrFile' => $stderrFile, - 'stdoutPipe' => $stdoutPipe, + 'process' => $process, + 'files' => $chunk, + 'buffer' => '', + 'inputFile' => $inputFile, + 'outputFile' => $outputFile, + 'stderrFile' => $stderrFile, + 'stdoutPipe' => $pipes[1], ]; } + // A worker's first line is how many of its files it still has to parse + // after hydrating the cached ones; summed, that is the progress total. + if ($emitProgress) { + $totalToParse = 0; + + foreach ($pending as $worker) { + $totalToParse += (int) fgets($worker['stdoutPipe']); + } + + $progressHandler->start($totalToParse); + } + + foreach ($pending as $worker) { + stream_set_blocking($worker['stdoutPipe'], false); + } + $nodes = []; $fileAnalyses = []; $anonymousClassNodes = []; @@ -162,15 +179,17 @@ public function extract( $data = fread($stdoutPipe, 8192); if ($data !== false && $data !== '') { - $workerFiles = $pending[$key]['files']; - $nextFileIdx = $pending[$key]['filesAdvanced']; - $lastFileIdx = min($nextFileIdx + substr_count($data, "\n"), count($workerFiles)); + // One chunk index per parsed file; a read may end mid-line. + $lines = explode("\n", $pending[$key]['buffer'] . $data); + $pending[$key]['buffer'] = array_pop($lines); - for (; $nextFileIdx < $lastFileIdx; $nextFileIdx++) { - $progressHandler?->advance($workerFiles[$nextFileIdx]); - } + foreach ($lines as $line) { + $file = $pending[$key]['files'][(int) $line] ?? null; - $pending[$key]['filesAdvanced'] = $nextFileIdx; + if ($file !== null) { + $progressHandler?->advance($file); + } + } $anyActivity = true; } diff --git a/src/Analyser/Parallel/WorkerProgressHandler.php b/src/Analyser/Parallel/WorkerProgressHandler.php index 0df022b9..b0b7e377 100644 --- a/src/Analyser/Parallel/WorkerProgressHandler.php +++ b/src/Analyser/Parallel/WorkerProgressHandler.php @@ -6,22 +6,35 @@ use Boundwize\StructArmed\Progress\ProgressHandlerInterface; +use function array_flip; use function fwrite; +/** + * Reports progress to the coordinator as one line per event: first the number + * of files this worker has to parse, then the chunk index of each parsed file. + */ final readonly class WorkerProgressHandler implements ProgressHandlerInterface { - /** @param resource $stream */ - public function __construct(private mixed $stream) + /** @var array */ + private array $fileIndexes; + + /** + * @param resource $stream + * @param list $files The worker's chunk, in the order the coordinator assigned it + */ + public function __construct(private mixed $stream, array $files) { + $this->fileIndexes = array_flip($files); } public function start(int $total): void { + fwrite($this->stream, $total . "\n"); } public function advance(string $file): void { - fwrite($this->stream, "\n"); + fwrite($this->stream, ($this->fileIndexes[$file] ?? -1) . "\n"); } public function finish(): void diff --git a/tests/Analyser/AnalysisNodeExtractorTest.php b/tests/Analyser/AnalysisNodeExtractorTest.php index b8b9aff3..3e70048a 100644 --- a/tests/Analyser/AnalysisNodeExtractorTest.php +++ b/tests/Analyser/AnalysisNodeExtractorTest.php @@ -7,6 +7,8 @@ use Boundwize\StructArmed\Analyser\AnalysisNodeExtractor; use Boundwize\StructArmed\Analyser\ClassNode; use Boundwize\StructArmed\Analyser\ExtractionResult; +use Boundwize\StructArmed\Cache\AnalysisResultCache; +use Boundwize\StructArmed\Cache\FileHashProvider; use Boundwize\StructArmed\LayerResolver\Resolvers\NamespaceLayerResolver; use Boundwize\StructArmed\Progress\ProgressHandlerInterface; use Boundwize\StructArmed\Tests\Support\TemporaryDirectoryCleanupTrait; @@ -199,4 +201,55 @@ public function finish(): void $this->assertCount(1, $advanced); $this->assertSame($file, $advanced[0]); } + + public function testExtractHydratesCachedFilesAndStartsProgressWithFilesNeedingParse(): void + { + $dir = $this->makeTemporaryDirectory('structarmed-extractor-test'); + $cacheDir = $this->makeTemporaryDirectory('structarmed-extractor-cache'); + $fooFile = $dir . '/Foo.php'; + $barFile = $dir . '/Bar.php'; + + file_put_contents($fooFile, ' new AnalysisNodeExtractor( + analysisResultCache: new AnalysisResultCache($dir, new FileHashProvider(), $cacheDir), + analysisNodeCacheNamespace: 'config', + ); + + $extractor()->extract([$fooFile, $barFile]); + + file_put_contents($fooFile, ' */ + public array $files = []; + + public function start(int $total): void + { + $this->total = $total; + } + + public function advance(string $file): void + { + $this->files[] = $file; + } + + public function finish(): void + { + } + }; + + $extractionResult = $extractor()->extract([$fooFile, $barFile], $progress); + + $this->assertSame(1, $progress->total); + $this->assertSame([$fooFile], $progress->files); + $this->assertCount(2, $extractionResult->classNodes); + // Cached nodes first, then the freshly parsed ones. + $this->assertStringEndsWith('\\Bar', $extractionResult->classNodes[0]->className); + $this->assertStringEndsWith('\\Foo', $extractionResult->classNodes[1]->className); + $this->assertCount(1, $extractionResult->classNodes[1]->methods); + } } diff --git a/tests/Analyser/Parallel/ParallelAnalysisNodeExtractorTest.php b/tests/Analyser/Parallel/ParallelAnalysisNodeExtractorTest.php index 0e81c299..6a689bb6 100644 --- a/tests/Analyser/Parallel/ParallelAnalysisNodeExtractorTest.php +++ b/tests/Analyser/Parallel/ParallelAnalysisNodeExtractorTest.php @@ -8,17 +8,20 @@ use Boundwize\StructArmed\Analyser\Parallel\ParallelAnalysisNodeExtractor; use Boundwize\StructArmed\Cache\AnalysisResultCache; use Boundwize\StructArmed\Cache\FileHashProvider; +use Boundwize\StructArmed\Progress\ProgressHandlerInterface; use Boundwize\StructArmed\Tests\Support\TemporaryDirectoryCleanupTrait; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; use RuntimeException; +use function array_map; use function bin2hex; use function file_put_contents; use function glob; use function is_dir; use function random_bytes; use function rmdir; +use function sort; use function sys_get_temp_dir; use function unlink; @@ -199,6 +202,73 @@ public function testWorkersStoreValidAnalysisNodeCacheEntriesWithScopedFileHashe $this->assertIsArray($changedFileCache->loadAnalysisNodes($barFile, 'config')); } + public function testWorkersHydrateCachedFilesAndReportOnlyParsedFilesAsProgress(): void + { + $dir = $this->makeTemporaryDirectory('structarmed-parallel-test'); + $cacheDir = $this->makeTemporaryDirectory('structarmed-parallel-cache'); + $fooFile = $dir . '/Foo.php'; + $barFile = $dir . '/Bar.php'; + + file_put_contents($fooFile, ' new ParallelAnalysisNodeExtractor( + basePath: $dir, + layers: [], + layerPatterns: [], + workerCount: 2, + cacheDirectory: $cacheDir, + analysisResultCache: new AnalysisResultCache($dir, new FileHashProvider(), $cacheDir), + analysisNodeCacheNamespace: 'config', + ); + + $extractor()->extract([$fooFile, $barFile]); + + file_put_contents($fooFile, ' */ + public array $files = []; + + public function start(int $total): void + { + $this->total = $total; + } + + public function advance(string $file): void + { + $this->files[] = $file; + } + + public function finish(): void + { + } + }; + + $extractionResult = $extractor()->extract([$fooFile, $barFile], $progress); + $classNames = array_map( + static fn (ClassNode $classNode): string => $classNode->className, + $extractionResult->classNodes + ); + sort($classNames); + + $this->assertCount(2, $classNames); + $this->assertStringEndsWith('\\Bar', $classNames[0]); + $this->assertStringEndsWith('\\Foo', $classNames[1]); + $this->assertSame(1, $progress->total); + $this->assertSame([$fooFile], $progress->files); + + $progress->total = -1; + $progress->files = []; + + $extractor()->extract([$fooFile, $barFile], $progress); + + $this->assertSame(0, $progress->total); + $this->assertSame([], $progress->files); + } + public function testExtractWithLayerPatternsUsesChainResolver(): void { $dir = $this->makeTemporaryDirectory('structarmed-parallel-test'); diff --git a/tests/Analyser/Parallel/WorkerProgressHandlerTest.php b/tests/Analyser/Parallel/WorkerProgressHandlerTest.php index dcc0803e..147b2788 100644 --- a/tests/Analyser/Parallel/WorkerProgressHandlerTest.php +++ b/tests/Analyser/Parallel/WorkerProgressHandlerTest.php @@ -14,17 +14,20 @@ final class WorkerProgressHandlerTest extends TestCase { use InMemoryStreamTrait; - public function testAdvanceWritesNewlineTokenToStream(): void + public function testWritesParseCountThenChunkIndexOfEachAdvancedFile(): void { $stream = $this->openMemoryStream(); - $workerProgressHandler = new WorkerProgressHandler($stream); + $workerProgressHandler = new WorkerProgressHandler( + $stream, + ['/path/Foo.php', '/path/Bar.php', '/path/Baz.php'] + ); $workerProgressHandler->start(2); + $workerProgressHandler->advance('/path/Baz.php'); $workerProgressHandler->advance('/path/Foo.php'); - $workerProgressHandler->advance('/path/Bar.php'); $workerProgressHandler->finish(); - $this->assertSame("\n\n", $this->streamContents($stream)); + $this->assertSame("2\n2\n0\n", $this->streamContents($stream)); } } From 7fd425d7e43ec27cfdac2cbc75c8b9f3967e20c4 Mon Sep 17 00:00:00 2001 From: Abdul Malik Ikhsan Date: Sun, 6 Sep 2026 13:03:47 +0700 Subject: [PATCH 2/2] mark WorkerProgressHandler as internal --- src/Analyser/Parallel/WorkerProgressHandler.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Analyser/Parallel/WorkerProgressHandler.php b/src/Analyser/Parallel/WorkerProgressHandler.php index b0b7e377..a8c2a973 100644 --- a/src/Analyser/Parallel/WorkerProgressHandler.php +++ b/src/Analyser/Parallel/WorkerProgressHandler.php @@ -12,6 +12,8 @@ /** * Reports progress to the coordinator as one line per event: first the number * of files this worker has to parse, then the chunk index of each parsed file. + * + * @internal */ final readonly class WorkerProgressHandler implements ProgressHandlerInterface {