diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index a0cfd6be90e..7a32cfbdef1 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -33,6 +33,9 @@ jobs: - 'e2e/parallel-unused-skips' - 'e2e/parallel-reflection-resolver' - 'e2e/parallel with space' + - 'e2e/parallel-worker-respawn' + - 'e2e/parallel-unfinished-jobs' + - 'e2e/parallel-system-errors-limit' - 'e2e/phpstan-container-bootstrap' - 'e2e/rules-summary-option' diff --git a/e2e/parallel-system-errors-limit/.gitignore b/e2e/parallel-system-errors-limit/.gitignore new file mode 100644 index 00000000000..61ead86667c --- /dev/null +++ b/e2e/parallel-system-errors-limit/.gitignore @@ -0,0 +1 @@ +/vendor diff --git a/e2e/parallel-system-errors-limit/composer.json b/e2e/parallel-system-errors-limit/composer.json new file mode 100644 index 00000000000..5468cd74606 --- /dev/null +++ b/e2e/parallel-system-errors-limit/composer.json @@ -0,0 +1,7 @@ +{ + "require": { + "php": "^8.1" + }, + "minimum-stability": "dev", + "prefer-stable": true +} diff --git a/e2e/parallel-system-errors-limit/expected-output.diff b/e2e/parallel-system-errors-limit/expected-output.diff new file mode 100644 index 00000000000..ef787becca4 --- /dev/null +++ b/e2e/parallel-system-errors-limit/expected-output.diff @@ -0,0 +1,156 @@ +[ERROR] Could not process "src/File05.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File06.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File07.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File08.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File09.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File10.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File11.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File12.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File13.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File14.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File15.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File16.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File17.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File18.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File19.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File20.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File21.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File22.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File23.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File24.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File25.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File26.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File27.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File28.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File29.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File30.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File31.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File32.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File33.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File34.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File35.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File36.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File37.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File38.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File39.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File40.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File41.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File42.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File43.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File44.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File45.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File46.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File47.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File48.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File49.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File50.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File51.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File52.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File53.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process "src/File54.php" file, due to: + "Syntax error, unexpected ';'". On line: 3 + + [ERROR] Could not process some files, due to: + "Reached system errors count limit of 50, exiting...". + + [ERROR] Could not process some files, due to: + "Some parallel jobs have not finished, results for 1 of 55 files are + missing". diff --git a/e2e/parallel-system-errors-limit/rector.php b/e2e/parallel-system-errors-limit/rector.php new file mode 100644 index 00000000000..2e8c8823f1a --- /dev/null +++ b/e2e/parallel-system-errors-limit/rector.php @@ -0,0 +1,20 @@ +paths([ + __DIR__ . '/src', + ]); + + // 4 valid files first, so the 50th syntax error arrives with the 54th chunk, when a worker's chunk budget + // (MAX_CHUNKS_PER_WORKER + 1) runs out while one job is left + $rectorConfig->parallel(maxNumberOfProcess: 1, jobSize: 1); + + $rectorConfig->rules([ + LongArrayToShortArrayRector::class, + ]); +}; diff --git a/e2e/parallel-system-errors-limit/src/File01.php b/e2e/parallel-system-errors-limit/src/File01.php new file mode 100644 index 00000000000..6ad04b7f539 --- /dev/null +++ b/e2e/parallel-system-errors-limit/src/File01.php @@ -0,0 +1,3 @@ +paths([ + __DIR__ . '/src', + ]); + + // the only worker ends on File3.php, so results for File3.php and File4.php are missing; + // the syntax error in File2.php must not hide that + $rectorConfig->parallel(maxNumberOfProcess: 1, jobSize: 1); + + $rectorConfig->rules([ + LongArrayToShortArrayRector::class, + ExitWorkerRector::class, + ]); +}; diff --git a/e2e/parallel-unfinished-jobs/src/File1.php b/e2e/parallel-unfinished-jobs/src/File1.php new file mode 100644 index 00000000000..6ff32900993 --- /dev/null +++ b/e2e/parallel-unfinished-jobs/src/File1.php @@ -0,0 +1,3 @@ +> + */ + public function getNodeTypes(): array + { + return [FuncCall::class]; + } + + /** + * @param FuncCall $node + */ + public function refactor(Node $node): ?Node + { + if ($this->isName($node, 'exit_worker')) { + exit(0); + } + + return null; + } +} diff --git a/e2e/parallel-worker-respawn/.gitignore b/e2e/parallel-worker-respawn/.gitignore new file mode 100644 index 00000000000..61ead86667c --- /dev/null +++ b/e2e/parallel-worker-respawn/.gitignore @@ -0,0 +1 @@ +/vendor diff --git a/e2e/parallel-worker-respawn/composer.json b/e2e/parallel-worker-respawn/composer.json new file mode 100644 index 00000000000..5468cd74606 --- /dev/null +++ b/e2e/parallel-worker-respawn/composer.json @@ -0,0 +1,7 @@ +{ + "require": { + "php": "^8.1" + }, + "minimum-stability": "dev", + "prefer-stable": true +} diff --git a/e2e/parallel-worker-respawn/expected-output.diff b/e2e/parallel-worker-respawn/expected-output.diff new file mode 100644 index 00000000000..59e4cf1eb40 --- /dev/null +++ b/e2e/parallel-worker-respawn/expected-output.diff @@ -0,0 +1,172 @@ +12 files with changes +===================== + +1) src/File01.php:1 + + ---------- begin diff ---------- +@@ Line 1 @@ + paths([ + __DIR__ . '/src', + ]); + + // a single worker with more single-file jobs than its chunk budget (MAX_CHUNKS_PER_WORKER + 1) has to be respawned mid-run + $rectorConfig->parallel(maxNumberOfProcess: 1, jobSize: 1); + + $rectorConfig->rules([ + LongArrayToShortArrayRector::class, + ]); +}; diff --git a/e2e/parallel-worker-respawn/src/File01.php b/e2e/parallel-worker-respawn/src/File01.php new file mode 100644 index 00000000000..6ff32900993 --- /dev/null +++ b/e2e/parallel-worker-respawn/src/File01.php @@ -0,0 +1,3 @@ +workerCommandLineFactory->create( @@ -185,11 +188,12 @@ function (array $json) use ( &$jobs, $postFileCallback, &$systemErrorsCount, - &$reachedInternalErrorsCountLimit, + &$reachedSystemErrorsCountLimit, $processIdentifier, &$fileChunksBudgetPerProcess, &$processSpawner, - &$totalChanged + &$totalChanged, + &$processedFilesCount ): void { /** @var array{ * total_changed: int, @@ -223,22 +227,24 @@ function (array $json) use ( } $postFileCallback($json[Bridge::FILES_COUNT]); + $processedFilesCount += $json[Bridge::FILES_COUNT]; $systemErrorsCount += $json[Bridge::SYSTEM_ERRORS_COUNT]; if ($systemErrorsCount >= self::SYSTEM_ERROR_LIMIT) { - $reachedInternalErrorsCountLimit = true; + $reachedSystemErrorsCountLimit = true; $this->processPool->quitAll(); + return; } - if ($fileChunksBudgetPerProcess[$processIdentifier] <= 0) { - // kill the current worker, and spawn a fresh one to free memory + if ($jobs === []) { $this->processPool->quitProcess($processIdentifier); - - ($processSpawner)(); return; } - if ($jobs === []) { + if ($fileChunksBudgetPerProcess[$processIdentifier] <= 0) { + // replace the current worker with a fresh one to free memory; spawn the replacement first, + // as quitting the last worker in the pool closes the server the replacement connects to + ($processSpawner)(); $this->processPool->quitProcess($processIdentifier); return; } @@ -290,6 +296,15 @@ function ($exitCode, string $stdErr) use (&$systemErrors, $processIdentifier): v )); } + // a worker can end without reporting its files, e.g. when killed by the OS, so missing results must always be reported + if ($processedFilesCount < $scheduledFilesCount) { + $systemErrors[] = new SystemError(sprintf( + 'Some parallel jobs have not finished, results for %d of %d files are missing', + $scheduledFilesCount - $processedFilesCount, + $scheduledFilesCount + )); + } + $mergedUsedSkips = []; foreach ($usedSkips as $skip => $paths) { $mergedUsedSkips[$skip] = array_keys($paths);