From 3100b6497e860bc0b700a79be3214d4b057e407d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksandr=20=C5=A0t=C5=A1epelin?= Date: Thu, 23 Apr 2026 11:09:59 +0300 Subject: [PATCH 1/3] fix: honor all stop-on-* flags in parallel mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously --stop-on-failure was the only flag honored in parallel mode, and it only cleared pending tests when the exit code was non-zero. The rest (--stop-on-defect, --stop-on-error, --stop-on-risky, --stop-on-incomplete, --stop-on-warning, --stop-on-notice, --stop-on-deprecation, --stop-on-skipped) were silently ignored. WrapperRunner now precomputes the set of progress-file characters that should trigger a stop (F, E, R, I, W, N, D, S) based on the configured flags, and scans each worker's progress file for them after every cycle. Once any matching outcome is seen, the pending queue is cleared. Up to N-1 in-flight tests (where N is --processes) can still complete after the first stop-worthy outcome — stopping them mid-run would risk corrupting shared state. Closes #1463. --- .../Parallel/Paratest/WrapperRunner.php | 87 ++++++++++++++++++- tests/.snapshots/success.txt | 19 +++- .../ParallelStopOn/HasDeprecationTest.php | 7 ++ tests/.tests/ParallelStopOn/HasErrorTest.php | 5 ++ .../.tests/ParallelStopOn/HasFailureTest.php | 3 + .../ParallelStopOn/HasIncompleteTest.php | 5 ++ tests/.tests/ParallelStopOn/HasNoticeTest.php | 7 ++ tests/.tests/ParallelStopOn/HasRiskyTest.php | 5 ++ .../.tests/ParallelStopOn/HasSkippedTest.php | 3 + .../.tests/ParallelStopOn/HasWarningTest.php | 7 ++ .../ParallelStopOn/Passing/Passing01Test.php | 3 + .../ParallelStopOn/Passing/Passing02Test.php | 3 + .../ParallelStopOn/Passing/Passing03Test.php | 3 + .../ParallelStopOn/Passing/Passing04Test.php | 3 + .../ParallelStopOn/Passing/Passing05Test.php | 3 + .../ParallelStopOn/Passing/Passing06Test.php | 3 + .../ParallelStopOn/Passing/Passing07Test.php | 3 + .../ParallelStopOn/Passing/Passing08Test.php | 3 + .../ParallelStopOn/Passing/Passing09Test.php | 3 + .../ParallelStopOn/Passing/Passing10Test.php | 3 + tests/Visual/Parallel.php | 30 +++++++ 21 files changed, 203 insertions(+), 5 deletions(-) create mode 100644 tests/.tests/ParallelStopOn/HasDeprecationTest.php create mode 100644 tests/.tests/ParallelStopOn/HasErrorTest.php create mode 100644 tests/.tests/ParallelStopOn/HasFailureTest.php create mode 100644 tests/.tests/ParallelStopOn/HasIncompleteTest.php create mode 100644 tests/.tests/ParallelStopOn/HasNoticeTest.php create mode 100644 tests/.tests/ParallelStopOn/HasRiskyTest.php create mode 100644 tests/.tests/ParallelStopOn/HasSkippedTest.php create mode 100644 tests/.tests/ParallelStopOn/HasWarningTest.php create mode 100644 tests/.tests/ParallelStopOn/Passing/Passing01Test.php create mode 100644 tests/.tests/ParallelStopOn/Passing/Passing02Test.php create mode 100644 tests/.tests/ParallelStopOn/Passing/Passing03Test.php create mode 100644 tests/.tests/ParallelStopOn/Passing/Passing04Test.php create mode 100644 tests/.tests/ParallelStopOn/Passing/Passing05Test.php create mode 100644 tests/.tests/ParallelStopOn/Passing/Passing06Test.php create mode 100644 tests/.tests/ParallelStopOn/Passing/Passing07Test.php create mode 100644 tests/.tests/ParallelStopOn/Passing/Passing08Test.php create mode 100644 tests/.tests/ParallelStopOn/Passing/Passing09Test.php create mode 100644 tests/.tests/ParallelStopOn/Passing/Passing10Test.php diff --git a/src/Plugins/Parallel/Paratest/WrapperRunner.php b/src/Plugins/Parallel/Paratest/WrapperRunner.php index 19ce3d1a4..2e817aaa3 100644 --- a/src/Plugins/Parallel/Paratest/WrapperRunner.php +++ b/src/Plugins/Parallel/Paratest/WrapperRunner.php @@ -33,12 +33,15 @@ use function array_merge; use function array_merge_recursive; use function array_shift; +use function array_unique; +use function array_values; use function assert; use function count; use function dirname; use function file_get_contents; use function max; use function realpath; +use function str_contains; use function str_starts_with; use function unlink; use function unserialize; @@ -107,6 +110,11 @@ final class WrapperRunner implements RunnerInterface /** @var non-empty-string[] */ private readonly array $parameters; + /** @var list|null */ + private ?array $stopOutcomeChars = null; + + private bool $stopRequested = false; + /** * The code coverage filter registry. */ @@ -210,10 +218,7 @@ private function assignAllPendingTests(): void $worker = $this->startWorker($token); } - if ( - $this->exitcode > 0 - && $this->options->configuration->stopOnFailure() - ) { + if ($this->shouldStop()) { $this->pending = []; } elseif (($pending = array_shift($this->pending)) !== null) { $worker->assign($pending); @@ -225,6 +230,80 @@ private function assignAllPendingTests(): void } } + private function shouldStop(): bool + { + if ($this->stopRequested) { + return true; + } + + $outcomes = $this->stopOutcomeChars(); + if ($outcomes === []) { + return false; + } + + foreach ($this->workers as $worker) { + $progress = file_get_contents($worker->progressFile->getPathname()); + if ($progress === false || $progress === '') { + continue; + } + + foreach ($outcomes as $outcome) { + if (str_contains($progress, $outcome)) { + return $this->stopRequested = true; + } + } + } + + return false; + } + + /** + * @return list + */ + private function stopOutcomeChars(): array + { + if ($this->stopOutcomeChars !== null) { + return $this->stopOutcomeChars; + } + + $configuration = $this->options->configuration; + $outcomes = []; + + if ($configuration->stopOnFailure() || $configuration->stopOnDefect()) { + $outcomes[] = 'F'; + } + + if ($configuration->stopOnError() || $configuration->stopOnDefect()) { + $outcomes[] = 'E'; + } + + if ($configuration->stopOnRisky() || $configuration->stopOnDefect()) { + $outcomes[] = 'R'; + } + + if ($configuration->stopOnIncomplete() || $configuration->stopOnDefect()) { + $outcomes[] = 'I'; + } + + if ($configuration->stopOnWarning()) { + $outcomes[] = 'W'; + } + + if ($configuration->stopOnNotice()) { + $outcomes[] = 'N'; + } + + if ($configuration->stopOnDeprecation()) { + $outcomes[] = 'D'; + } + + if ($configuration->stopOnSkipped()) { + $outcomes[] = 'S'; + } + + return $this->stopOutcomeChars = array_values(array_unique($outcomes)); + } + private function flushWorker(WrapperWorker $worker): void { $this->exitcode = max($this->exitcode, $worker->getExitCode()); diff --git a/tests/.snapshots/success.txt b/tests/.snapshots/success.txt index 0e38f50f0..68bf017b6 100644 --- a/tests/.snapshots/success.txt +++ b/tests/.snapshots/success.txt @@ -1908,6 +1908,23 @@ ✓ parallel ✓ a parallel test can extend another test with same name ✓ parallel reports invalid datasets as failures + ✓ parallel stop-on-* fixture runs every case without a stop flag with ('fails') + ✓ parallel stop-on-* fixture runs every case without a stop flag with ('errors') + ✓ parallel stop-on-* fixture runs every case without a stop flag with ('is risky') + ✓ parallel stop-on-* fixture runs every case without a stop flag with ('is incomplete') + ✓ parallel stop-on-* fixture runs every case without a stop flag with ('warns') + ✓ parallel stop-on-* fixture runs every case without a stop flag with ('notices') + ✓ parallel stop-on-* fixture runs every case without a stop flag with ('deprecates') + ✓ parallel stop-on-* fixture runs every case without a stop flag with ('is skipped') + ✓ parallel honors --stop-on-* with dataset "failure" + ✓ parallel honors --stop-on-* with dataset "defect" + ✓ parallel honors --stop-on-* with dataset "error" + ✓ parallel honors --stop-on-* with dataset "risky" + ✓ parallel honors --stop-on-* with dataset "incomplete" + ✓ parallel honors --stop-on-* with dataset "warning" + ✓ parallel honors --stop-on-* with dataset "notice" + ✓ parallel honors --stop-on-* with dataset "deprecation" + ✓ parallel honors --stop-on-* with dataset "skipped" PASS Tests\Visual\ParallelNestedDatasets ✓ parallel loads nested datasets from nested directories @@ -1941,4 +1958,4 @@ ✓ pass with dataset with ('my-datas-set-value') ✓ within describe → pass with dataset with ('my-datas-set-value') - Tests: 2 deprecated, 4 warnings, 5 incomplete, 2 notices, 40 todos, 35 skipped, 1330 passed (3010 assertions) \ No newline at end of file + Tests: 2 deprecated, 4 warnings, 5 incomplete, 2 notices, 40 todos, 35 skipped, 1347 passed (3027 assertions) \ No newline at end of file diff --git a/tests/.tests/ParallelStopOn/HasDeprecationTest.php b/tests/.tests/ParallelStopOn/HasDeprecationTest.php new file mode 100644 index 000000000..47e031b06 --- /dev/null +++ b/tests/.tests/ParallelStopOn/HasDeprecationTest.php @@ -0,0 +1,7 @@ +toBeTrue(); +}); diff --git a/tests/.tests/ParallelStopOn/HasErrorTest.php b/tests/.tests/ParallelStopOn/HasErrorTest.php new file mode 100644 index 000000000..7c9d0ed5d --- /dev/null +++ b/tests/.tests/ParallelStopOn/HasErrorTest.php @@ -0,0 +1,5 @@ +assertTrue(false); diff --git a/tests/.tests/ParallelStopOn/HasIncompleteTest.php b/tests/.tests/ParallelStopOn/HasIncompleteTest.php new file mode 100644 index 000000000..9c9fff147 --- /dev/null +++ b/tests/.tests/ParallelStopOn/HasIncompleteTest.php @@ -0,0 +1,5 @@ +markTestIncomplete('work in progress'); +}); diff --git a/tests/.tests/ParallelStopOn/HasNoticeTest.php b/tests/.tests/ParallelStopOn/HasNoticeTest.php new file mode 100644 index 000000000..552f11b23 --- /dev/null +++ b/tests/.tests/ParallelStopOn/HasNoticeTest.php @@ -0,0 +1,7 @@ +toBeTrue(); +}); diff --git a/tests/.tests/ParallelStopOn/HasRiskyTest.php b/tests/.tests/ParallelStopOn/HasRiskyTest.php new file mode 100644 index 000000000..5519b1ad0 --- /dev/null +++ b/tests/.tests/ParallelStopOn/HasRiskyTest.php @@ -0,0 +1,5 @@ +skip('skipping'); diff --git a/tests/.tests/ParallelStopOn/HasWarningTest.php b/tests/.tests/ParallelStopOn/HasWarningTest.php new file mode 100644 index 000000000..8fae1020d --- /dev/null +++ b/tests/.tests/ParallelStopOn/HasWarningTest.php @@ -0,0 +1,7 @@ +toBeTrue(); +}); diff --git a/tests/.tests/ParallelStopOn/Passing/Passing01Test.php b/tests/.tests/ParallelStopOn/Passing/Passing01Test.php new file mode 100644 index 000000000..8dc2e1fb5 --- /dev/null +++ b/tests/.tests/ParallelStopOn/Passing/Passing01Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/.tests/ParallelStopOn/Passing/Passing02Test.php b/tests/.tests/ParallelStopOn/Passing/Passing02Test.php new file mode 100644 index 000000000..86045d01e --- /dev/null +++ b/tests/.tests/ParallelStopOn/Passing/Passing02Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/.tests/ParallelStopOn/Passing/Passing03Test.php b/tests/.tests/ParallelStopOn/Passing/Passing03Test.php new file mode 100644 index 000000000..057c6070f --- /dev/null +++ b/tests/.tests/ParallelStopOn/Passing/Passing03Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/.tests/ParallelStopOn/Passing/Passing04Test.php b/tests/.tests/ParallelStopOn/Passing/Passing04Test.php new file mode 100644 index 000000000..883200ba6 --- /dev/null +++ b/tests/.tests/ParallelStopOn/Passing/Passing04Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/.tests/ParallelStopOn/Passing/Passing05Test.php b/tests/.tests/ParallelStopOn/Passing/Passing05Test.php new file mode 100644 index 000000000..8a050a111 --- /dev/null +++ b/tests/.tests/ParallelStopOn/Passing/Passing05Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/.tests/ParallelStopOn/Passing/Passing06Test.php b/tests/.tests/ParallelStopOn/Passing/Passing06Test.php new file mode 100644 index 000000000..ceef2a2fb --- /dev/null +++ b/tests/.tests/ParallelStopOn/Passing/Passing06Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/.tests/ParallelStopOn/Passing/Passing07Test.php b/tests/.tests/ParallelStopOn/Passing/Passing07Test.php new file mode 100644 index 000000000..1f0a6b0f6 --- /dev/null +++ b/tests/.tests/ParallelStopOn/Passing/Passing07Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/.tests/ParallelStopOn/Passing/Passing08Test.php b/tests/.tests/ParallelStopOn/Passing/Passing08Test.php new file mode 100644 index 000000000..bae06622c --- /dev/null +++ b/tests/.tests/ParallelStopOn/Passing/Passing08Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/.tests/ParallelStopOn/Passing/Passing09Test.php b/tests/.tests/ParallelStopOn/Passing/Passing09Test.php new file mode 100644 index 000000000..bcc6b5e14 --- /dev/null +++ b/tests/.tests/ParallelStopOn/Passing/Passing09Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/.tests/ParallelStopOn/Passing/Passing10Test.php b/tests/.tests/ParallelStopOn/Passing/Passing10Test.php new file mode 100644 index 000000000..edc06d046 --- /dev/null +++ b/tests/.tests/ParallelStopOn/Passing/Passing10Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/Visual/Parallel.php b/tests/Visual/Parallel.php index d497eef3c..1fb444168 100644 --- a/tests/Visual/Parallel.php +++ b/tests/Visual/Parallel.php @@ -47,3 +47,33 @@ ->toContain('Tests: 1 failed, 1 passed (1 assertions)') ->toContain('Parallel: 3 processes'); })->skipOnWindows(); + +test('parallel stop-on-* fixture runs every case without a stop flag', function (string $testName) use ($run) { + $output = $run('--filter', "($testName|passes)", 'tests/.tests/ParallelStopOn'); + expect($output)->toContain('10 passed'); +})->with([ + 'fails', + 'errors', + 'is risky', + 'is incomplete', + 'warns', + 'notices', + 'deprecates', + 'is skipped', +])->skipOnWindows(); + +test('parallel honors --stop-on-*', function (string $flag, string $testName) use ($run) { + $output = $run('--filter', "($testName|passes)", $flag, 'tests/.tests/ParallelStopOn'); + preg_match('/(\d+) passed/', $output, $matches); + expect((int) ($matches[1] ?? 0))->toBeLessThan(10); +})->with([ + 'failure' => ['--stop-on-failure', 'fails'], + 'defect' => ['--stop-on-defect', 'fails'], + 'error' => ['--stop-on-error', 'errors'], + 'risky' => ['--stop-on-risky', 'is risky'], + 'incomplete' => ['--stop-on-incomplete', 'is incomplete'], + 'warning' => ['--stop-on-warning', 'warns'], + 'notice' => ['--stop-on-notice', 'notices'], + 'deprecation' => ['--stop-on-deprecation', 'deprecates'], + 'skipped' => ['--stop-on-skipped', 'is skipped'], +])->skipOnWindows(); From 3f5d5cce09e00cea9db3858cc91d498217897205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksandr=20=C5=A0t=C5=A1epelin?= Date: Thu, 12 Nov 2026 00:09:11 +0200 Subject: [PATCH 2/3] fix: preserve parallel stop conditions on Pest 5 --- bin/worker.php | 7 + src/Plugins/Parallel.php | 3 + src/Plugins/Parallel/Handlers/StopOn.php | 76 ++++++ .../Parallel/Paratest/StopConditions.php | 216 ++++++++++++++++++ .../Parallel/Paratest/WrapperRunner.php | 103 ++------- 5 files changed, 317 insertions(+), 88 deletions(-) create mode 100644 src/Plugins/Parallel/Handlers/StopOn.php create mode 100644 src/Plugins/Parallel/Paratest/StopConditions.php diff --git a/bin/worker.php b/bin/worker.php index 8a4574dbd..c4bce8562 100644 --- a/bin/worker.php +++ b/bin/worker.php @@ -6,8 +6,10 @@ use ParaTest\WrapperRunner\WrapperWorker; use Pest\Kernel; use Pest\Plugins\Actions\CallsHandleArguments; +use Pest\Plugins\Parallel\Paratest\StopConditions; use Pest\Support\Container; use Pest\TestSuite; +use PHPUnit\TextUI\Configuration\Registry as ConfigurationRegistry; use Symfony\Component\Console\Input\ArgvInput; use Symfony\Component\Console\Output\ConsoleOutput; use Symfony\Component\Console\Output\OutputInterface; @@ -99,6 +101,8 @@ (int) ($getopt['testdox-columns'] ?? null), ); + $stopConditions = null; + while (true) { if (feof(STDIN)) { $application->end(); @@ -113,6 +117,9 @@ $exitCode = $application->runTest(realpath(trim($testPath))); + $stopConditions ??= new StopConditions(ConfigurationRegistry::get()); + $stopConditions->write($getopt['test-result-file'].'.stop'); + fwrite($statusFile, (string) $exitCode); fflush($statusFile); } diff --git a/src/Plugins/Parallel.php b/src/Plugins/Parallel.php index cd9ecab5d..b2a5d1fb7 100644 --- a/src/Plugins/Parallel.php +++ b/src/Plugins/Parallel.php @@ -29,6 +29,7 @@ final class Parallel implements HandlesArguments Parallel\Handlers\Parallel::class, Parallel\Handlers\Pest::class, Parallel\Handlers\Laravel::class, + Parallel\Handlers\StopOn::class, ]; /** @@ -148,6 +149,8 @@ private function paratestCommand(): Application $command->setName('Pest'); $command->setVersion(version()); + Parallel\Handlers\StopOn::configureOptions($command->find('paratest')->getDefinition()); + return $command; } diff --git a/src/Plugins/Parallel/Handlers/StopOn.php b/src/Plugins/Parallel/Handlers/StopOn.php new file mode 100644 index 000000000..cc06e2f0d --- /dev/null +++ b/src/Plugins/Parallel/Handlers/StopOn.php @@ -0,0 +1,76 @@ + $argument) { + [$option, $value] = array_pad(explode('=', $argument, 2), 2, ''); + + if (in_array($option, self::OPTIONS, true) + && in_array($value, ['', '0'], true)) { + $arguments[$index] = $option.'=1'; + } + } + + return $arguments; + } + + public function handleWorkerArguments(array $arguments): array + { + $arguments = array_values($arguments); + + foreach ($arguments as $index => $argument) { + if (! in_array($argument, self::OPTIONS, true) + || ! isset($arguments[$index + 1]) + || str_starts_with($arguments[$index + 1], '--')) { + continue; + } + + $arguments[$index] .= '='.$arguments[$index + 1]; + unset($arguments[$index + 1]); + } + + return array_values($arguments); + } + + public static function configureOptions(InputDefinition $definition): void + { + $options = $definition->getOptions(); + + foreach (self::OPTIONS as $argument) { + $name = substr($argument, 2); + $option = $options[$name]; + $options[$name] = new InputOption( + $name, + $option->getShortcut(), + InputOption::VALUE_OPTIONAL, + $option->getDescription(), + false, + ); + } + + $definition->setOptions($options); + } +} diff --git a/src/Plugins/Parallel/Paratest/StopConditions.php b/src/Plugins/Parallel/Paratest/StopConditions.php new file mode 100644 index 000000000..f5a197515 --- /dev/null +++ b/src/Plugins/Parallel/Paratest/StopConditions.php @@ -0,0 +1,216 @@ + + */ + private readonly array $thresholds; + + /** + * @var array> + */ + private array $workerCounts = []; + + /** + * @var array + */ + private array $totals = []; + + /** + * @var array> + */ + private array $issueIds = []; + + public function __construct(private readonly Configuration $configuration) + { + $this->thresholds = array_filter([ + 'defect' => $configuration->stopOnDefectThreshold(), + 'error' => $configuration->stopOnErrorThreshold(), + 'failure' => $configuration->stopOnFailureThreshold(), + 'warning' => $configuration->stopOnWarningThreshold(), + 'risky' => $configuration->stopOnRiskyThreshold(), + 'deprecation' => $configuration->stopOnDeprecationThreshold(), + 'notice' => $configuration->stopOnNoticeThreshold(), + 'incomplete' => $configuration->stopOnIncompleteThreshold(), + 'skipped' => $configuration->stopOnSkippedThreshold(), + ], static fn (int $threshold): bool => $threshold > 0); + } + + public function write(string $filename): void + { + if ($this->thresholds === []) { + return; + } + + $result = TestResultFacade::result(); + $state = json_encode([ + 'counts' => $this->counts($result), + 'issues' => $this->issues($result), + ], JSON_THROW_ON_ERROR); + + if (file_put_contents($filename.'.tmp', $state) === false + || ! rename($filename.'.tmp', $filename)) { + throw new RuntimeException(sprintf('Unable to write the parallel stop counts to [%s].', $filename)); + } + } + + public function read(string $filename): void + { + if ($this->thresholds === []) { + return; + } + + $contents = file_get_contents($filename); + + if ($contents === false) { + throw new RuntimeException(sprintf('Unable to read the parallel stop counts from [%s].', $filename)); + } + + $state = json_decode($contents, true, flags: JSON_THROW_ON_ERROR); + + if (! is_array($state) + || ! isset($state['counts'], $state['issues']) + || ! is_array($state['counts']) + || ! is_array($state['issues'])) { + throw new RuntimeException(sprintf('Invalid parallel stop counts in [%s].', $filename)); + } + + $this->addCounts($filename, $state['counts']); + $this->addIssues($filename, $state['issues']); + } + + public function shouldStop(): bool + { + return array_any($this->thresholds, function (int $threshold, string $outcome): bool { + $issueType = $outcome === 'defect' ? 'warning' : $outcome; + $count = ($this->totals[$outcome] ?? 0) + count($this->issueIds[$issueType] ?? []); + + return $count >= $threshold; + }); + } + + /** + * @return array + */ + private function counts(TestResult $result): array + { + $counts = [ + 'error' => $result->numberOfTestErroredEvents(), + 'failure' => $result->numberOfTestFailedEvents(), + 'warning' => $result->numberOfPhpunitWarnings(), + 'risky' => $result->numberOfTestsWithTestConsideredRiskyEvents(), + 'deprecation' => $this->numberOfDeprecations(), + 'notice' => 0, + 'incomplete' => $result->numberOfTestMarkedIncompleteEvents(), + 'skipped' => $result->numberOfTestSkippedEvents(), + ]; + $counts['defect'] = $counts['error'] + $counts['failure'] + $counts['warning'] + $counts['risky']; + + return $counts; + } + + /** + * @return array<'warning'|'notice', list> + */ + private function issues(TestResult $result): array + { + $issues = []; + + if (isset($this->thresholds['warning']) || isset($this->thresholds['defect'])) { + $issues['warning'] = array_merge( + $this->issueKeys($result->warnings(), 'user'), + $this->issueKeys($result->phpWarnings(), 'php'), + ); + } + + if (isset($this->thresholds['notice'])) { + $issues['notice'] = array_merge( + $this->issueKeys($result->notices(), 'user'), + $this->issueKeys($result->phpNotices(), 'php'), + ); + } + + return $issues; + } + + /** + * @return non-negative-int + */ + private function numberOfDeprecations(): int + { + $deprecations = DeprecationCollector::filteredDeprecations(); + + if ($this->configuration->hasSpecificDeprecationToStopOn()) { + $message = $this->configuration->specificDeprecationToStopOn(); + $deprecations = array_filter($deprecations, static fn (string $deprecation): bool => str_contains($deprecation, $message)); + } + + return count($deprecations); + } + + /** + * @param array $counts + */ + private function addCounts(string $filename, array $counts): void + { + foreach (array_keys($this->thresholds) as $outcome) { + if (! isset($counts[$outcome]) || ! is_int($counts[$outcome]) || $counts[$outcome] < 0) { + throw new RuntimeException(sprintf('Invalid parallel stop counts in [%s].', $filename)); + } + + $previous = $this->workerCounts[$filename][$outcome] ?? 0; + $increase = $counts[$outcome] - $previous; + + if ($increase < 0) { + throw new RuntimeException(sprintf('Parallel stop counts decreased in [%s].', $filename)); + } + + $this->totals[$outcome] = ($this->totals[$outcome] ?? 0) + $increase; + $this->workerCounts[$filename][$outcome] = $counts[$outcome]; + } + } + + /** + * @param array $issues + */ + private function addIssues(string $filename, array $issues): void + { + foreach ($issues as $outcome => $ids) { + if (! in_array($outcome, ['warning', 'notice'], true) || ! is_array($ids)) { + throw new RuntimeException(sprintf('Invalid parallel stop issues in [%s].', $filename)); + } + + foreach ($ids as $id) { + if (! is_string($id)) { + throw new RuntimeException(sprintf('Invalid parallel stop issue in [%s].', $filename)); + } + + $this->issueIds[$outcome][$id] = true; + } + } + } + + /** + * @param list $issues + * @return list + */ + private function issueKeys(array $issues, string $origin): array + { + return array_map( + static fn (Issue $issue): string => hash('sha256', implode(':', [$origin, $issue->file(), $issue->line(), $issue->description()])), + $issues, + ); + } +} diff --git a/src/Plugins/Parallel/Paratest/WrapperRunner.php b/src/Plugins/Parallel/Paratest/WrapperRunner.php index 9d768378f..676af83f3 100644 --- a/src/Plugins/Parallel/Paratest/WrapperRunner.php +++ b/src/Plugins/Parallel/Paratest/WrapperRunner.php @@ -42,8 +42,6 @@ use function array_merge; use function array_merge_recursive; use function array_shift; -use function array_unique; -use function array_values; use function assert; use function bin2hex; use function count; @@ -57,7 +55,6 @@ use function realpath; use function rmdir; use function sprintf; -use function str_contains; use function str_starts_with; use function unlink; use function unserialize; @@ -81,8 +78,6 @@ final class WrapperRunner implements RunnerInterface /** @var list */ private array $pending = []; - private int $exitcode = -1; - /** @var array */ private array $workers = []; @@ -125,10 +120,7 @@ final class WrapperRunner implements RunnerInterface /** @var non-empty-string[] */ private readonly array $parameters; - /** @var list|null */ - private ?array $stopOutcomeChars = null; - - private bool $stopRequested = false; + private readonly StopConditions $stopConditions; private CodeCoverageFilterRegistry $codeCoverageFilterRegistry; @@ -141,6 +133,7 @@ public function __construct( ) { $this->printer = new ResultPrinter($output, $options); $this->timer = new Timer; + $this->stopConditions = new StopConditions($options->configuration); $wrapper = realpath( dirname(__DIR__, 4).DIRECTORY_SEPARATOR.'bin'.DIRECTORY_SEPARATOR.'worker.php', @@ -256,14 +249,18 @@ private function assignAllPendingTests(): void $this->flushWorker($worker); + if ($this->stopConditions->shouldStop()) { + $this->pending = []; + + break; + } + if ($batchSize !== 0 && $this->batches[$token] === $batchSize) { $this->destroyWorker($token); $worker = $this->startWorker($token); } - if ($this->shouldStop()) { - $this->pending = []; - } elseif (($pending = array_shift($this->pending)) !== null) { + if (($pending = array_shift($this->pending)) !== null) { $worker->assign($pending); $this->batches[$token]++; } @@ -273,80 +270,6 @@ private function assignAllPendingTests(): void } } - private function shouldStop(): bool - { - if ($this->stopRequested) { - return true; - } - - $outcomes = $this->stopOutcomeChars(); - if ($outcomes === []) { - return false; - } - - foreach ($this->workers as $worker) { - $progress = file_get_contents($worker->progressFile->getPathname()); - if ($progress === false || $progress === '') { - continue; - } - - foreach ($outcomes as $outcome) { - if (str_contains($progress, $outcome)) { - return $this->stopRequested = true; - } - } - } - - return false; - } - - /** - * @return list - */ - private function stopOutcomeChars(): array - { - if ($this->stopOutcomeChars !== null) { - return $this->stopOutcomeChars; - } - - $configuration = $this->options->configuration; - $outcomes = []; - - if ($configuration->stopOnFailure() || $configuration->stopOnDefect()) { - $outcomes[] = 'F'; - } - - if ($configuration->stopOnError() || $configuration->stopOnDefect()) { - $outcomes[] = 'E'; - } - - if ($configuration->stopOnRisky() || $configuration->stopOnDefect()) { - $outcomes[] = 'R'; - } - - if ($configuration->stopOnIncomplete() || $configuration->stopOnDefect()) { - $outcomes[] = 'I'; - } - - if ($configuration->stopOnWarning()) { - $outcomes[] = 'W'; - } - - if ($configuration->stopOnNotice()) { - $outcomes[] = 'N'; - } - - if ($configuration->stopOnDeprecation()) { - $outcomes[] = 'D'; - } - - if ($configuration->stopOnSkipped()) { - $outcomes[] = 'S'; - } - - return $this->stopOutcomeChars = array_values(array_unique($outcomes)); - } - private function flushWorker(WrapperWorker $worker): void { if ($worker->hasExecutedTests()) { @@ -354,6 +277,7 @@ private function flushWorker(WrapperWorker $worker): void if ($testResultFile !== '') { $this->requiredTestResultFiles[$testResultFile] = true; + $this->stopConditions->read($testResultFile.'.stop'); } if (isset($worker->coverageFile) && $worker->coverageFile->getPathname() !== '') { @@ -361,7 +285,6 @@ private function flushWorker(WrapperWorker $worker): void } } - $this->exitcode = max($this->exitcode, $worker->getExitCode()); $this->printer->printFeedback( $worker->progressFile, $worker->unexpectedOutputFile, @@ -580,7 +503,11 @@ private function complete(TestResult $testResultSum): int $this->clearFiles($this->statusFiles); $this->clearFiles($this->progressFiles); $this->clearFiles($this->unexpectedOutputFiles); - $this->clearFiles($this->testResultFiles); + + foreach ($this->testResultFiles as $file) { + $this->clearFiles([$file, new SplFileInfo($file->getPathname().'.stop')]); + } + $this->clearFiles($this->resultCacheFiles); $this->clearFiles($this->coverageFiles); $this->clearFiles($this->junitFiles); From 4135a02c42baf28359ff115c7da3d1847836f054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksandr=20=C5=A0t=C5=A1epelin?= Date: Thu, 12 Nov 2026 00:09:21 +0200 Subject: [PATCH 3/3] test: expand parallel stop-on regression coverage --- tests/.snapshots/success.txt | 87 ++++++++-- .../.tests/ParallelStopOn/HasFailureTest.php | 3 - tests/Fixtures/Suites/ParallelStopOn.xml | 8 + .../ParallelStopOn/HasDeprecationTest.php | 0 .../Suites}/ParallelStopOn/HasErrorTest.php | 0 .../Suites/ParallelStopOn/HasFailureTest.php | 3 + .../ParallelStopOn/HasIncompleteTest.php | 0 .../Suites}/ParallelStopOn/HasNoticeTest.php | 0 .../Suites}/ParallelStopOn/HasRiskyTest.php | 2 +- .../ParallelStopOn/HasSharedIssueTest.php | 7 + .../Suites}/ParallelStopOn/HasSkippedTest.php | 0 .../Suites}/ParallelStopOn/HasWarningTest.php | 0 .../ParallelStopOn/MixedOutcomesTest.php | 21 +++ .../More/HasDeprecationTest.php | 7 + .../ParallelStopOn/More/HasErrorTest.php | 5 + .../ParallelStopOn/More/HasFailureTest.php | 3 + .../ParallelStopOn/More/HasIncompleteTest.php | 5 + .../ParallelStopOn/More/HasNoticeTest.php | 7 + .../ParallelStopOn/More/HasRiskyTest.php | 5 + .../More/HasSharedIssueTest.php | 7 + .../ParallelStopOn/More/HasSkippedTest.php | 3 + .../ParallelStopOn/More/HasWarningTest.php | 7 + .../ParallelStopOn/Passing/Passing01Test.php | 0 .../ParallelStopOn/Passing/Passing02Test.php | 0 .../ParallelStopOn/Passing/Passing03Test.php | 0 .../ParallelStopOn/Passing/Passing04Test.php | 0 .../ParallelStopOn/Passing/Passing05Test.php | 0 .../ParallelStopOn/Passing/Passing06Test.php | 0 .../ParallelStopOn/Passing/Passing07Test.php | 0 .../ParallelStopOn/Passing/Passing08Test.php | 0 .../ParallelStopOn/Passing/Passing09Test.php | 0 .../ParallelStopOn/Passing/Passing10Test.php | 0 .../Fixtures/Suites/ParallelStopOnFailure.php | 20 +++ .../Suites/ParallelStopOnSharedIssue.php | 7 + tests/Visual/Parallel.php | 30 ---- tests/Visual/ParallelStopOn.php | 164 ++++++++++++++++++ 36 files changed, 349 insertions(+), 52 deletions(-) delete mode 100644 tests/.tests/ParallelStopOn/HasFailureTest.php create mode 100644 tests/Fixtures/Suites/ParallelStopOn.xml rename tests/{.tests => Fixtures/Suites}/ParallelStopOn/HasDeprecationTest.php (100%) rename tests/{.tests => Fixtures/Suites}/ParallelStopOn/HasErrorTest.php (100%) create mode 100644 tests/Fixtures/Suites/ParallelStopOn/HasFailureTest.php rename tests/{.tests => Fixtures/Suites}/ParallelStopOn/HasIncompleteTest.php (100%) rename tests/{.tests => Fixtures/Suites}/ParallelStopOn/HasNoticeTest.php (100%) rename tests/{.tests => Fixtures/Suites}/ParallelStopOn/HasRiskyTest.php (65%) create mode 100644 tests/Fixtures/Suites/ParallelStopOn/HasSharedIssueTest.php rename tests/{.tests => Fixtures/Suites}/ParallelStopOn/HasSkippedTest.php (100%) rename tests/{.tests => Fixtures/Suites}/ParallelStopOn/HasWarningTest.php (100%) create mode 100644 tests/Fixtures/Suites/ParallelStopOn/MixedOutcomesTest.php create mode 100644 tests/Fixtures/Suites/ParallelStopOn/More/HasDeprecationTest.php create mode 100644 tests/Fixtures/Suites/ParallelStopOn/More/HasErrorTest.php create mode 100644 tests/Fixtures/Suites/ParallelStopOn/More/HasFailureTest.php create mode 100644 tests/Fixtures/Suites/ParallelStopOn/More/HasIncompleteTest.php create mode 100644 tests/Fixtures/Suites/ParallelStopOn/More/HasNoticeTest.php create mode 100644 tests/Fixtures/Suites/ParallelStopOn/More/HasRiskyTest.php create mode 100644 tests/Fixtures/Suites/ParallelStopOn/More/HasSharedIssueTest.php create mode 100644 tests/Fixtures/Suites/ParallelStopOn/More/HasSkippedTest.php create mode 100644 tests/Fixtures/Suites/ParallelStopOn/More/HasWarningTest.php rename tests/{.tests => Fixtures/Suites}/ParallelStopOn/Passing/Passing01Test.php (100%) rename tests/{.tests => Fixtures/Suites}/ParallelStopOn/Passing/Passing02Test.php (100%) rename tests/{.tests => Fixtures/Suites}/ParallelStopOn/Passing/Passing03Test.php (100%) rename tests/{.tests => Fixtures/Suites}/ParallelStopOn/Passing/Passing04Test.php (100%) rename tests/{.tests => Fixtures/Suites}/ParallelStopOn/Passing/Passing05Test.php (100%) rename tests/{.tests => Fixtures/Suites}/ParallelStopOn/Passing/Passing06Test.php (100%) rename tests/{.tests => Fixtures/Suites}/ParallelStopOn/Passing/Passing07Test.php (100%) rename tests/{.tests => Fixtures/Suites}/ParallelStopOn/Passing/Passing08Test.php (100%) rename tests/{.tests => Fixtures/Suites}/ParallelStopOn/Passing/Passing09Test.php (100%) rename tests/{.tests => Fixtures/Suites}/ParallelStopOn/Passing/Passing10Test.php (100%) create mode 100644 tests/Fixtures/Suites/ParallelStopOnFailure.php create mode 100644 tests/Fixtures/Suites/ParallelStopOnSharedIssue.php create mode 100644 tests/Visual/ParallelStopOn.php diff --git a/tests/.snapshots/success.txt b/tests/.snapshots/success.txt index 506b97b33..eca0d4843 100644 --- a/tests/.snapshots/success.txt +++ b/tests/.snapshots/success.txt @@ -2279,23 +2279,6 @@ ✓ parallel ✓ a parallel test can extend another test with same name ✓ parallel reports invalid datasets as failures - ✓ parallel stop-on-* fixture runs every case without a stop flag with ('fails') - ✓ parallel stop-on-* fixture runs every case without a stop flag with ('errors') - ✓ parallel stop-on-* fixture runs every case without a stop flag with ('is risky') - ✓ parallel stop-on-* fixture runs every case without a stop flag with ('is incomplete') - ✓ parallel stop-on-* fixture runs every case without a stop flag with ('warns') - ✓ parallel stop-on-* fixture runs every case without a stop flag with ('notices') - ✓ parallel stop-on-* fixture runs every case without a stop flag with ('deprecates') - ✓ parallel stop-on-* fixture runs every case without a stop flag with ('is skipped') - ✓ parallel honors --stop-on-* with dataset "failure" - ✓ parallel honors --stop-on-* with dataset "defect" - ✓ parallel honors --stop-on-* with dataset "error" - ✓ parallel honors --stop-on-* with dataset "risky" - ✓ parallel honors --stop-on-* with dataset "incomplete" - ✓ parallel honors --stop-on-* with dataset "warning" - ✓ parallel honors --stop-on-* with dataset "notice" - ✓ parallel honors --stop-on-* with dataset "deprecation" - ✓ parallel honors --stop-on-* with dataset "skipped" ✓ parallel lets extensions register event subscribers ✓ parallel can have multiple exclude-groups ✓ parallel can have multiple groups @@ -2311,6 +2294,74 @@ PASS Tests\Visual\ParallelNestedDatasets ✓ parallel loads nested datasets from nested directories + PASS Tests\Visual\ParallelStopOn + ✓ parallel stop-on fixtures run completely without a stop flag with dataset "failure" + ✓ parallel stop-on fixtures run completely without a stop flag with dataset "error" + ✓ parallel stop-on fixtures run completely without a stop flag with dataset "risky" + ✓ parallel stop-on fixtures run completely without a stop flag with dataset "incomplete" + ✓ parallel stop-on fixtures run completely without a stop flag with dataset "warning" + ✓ parallel stop-on fixtures run completely without a stop flag with dataset "notice" + ✓ parallel stop-on fixtures run completely without a stop flag with dataset "deprecation" + ✓ parallel stop-on fixtures run completely without a stop flag with dataset "skipped" + ✓ parallel honors stop-on flags with worker recycling with dataset "failure" / (0) + ✓ parallel honors stop-on flags with worker recycling with dataset "failure" / (1) + ✓ parallel honors stop-on flags with worker recycling with dataset "defect" / (0) + ✓ parallel honors stop-on flags with worker recycling with dataset "defect" / (1) + ✓ parallel honors stop-on flags with worker recycling with dataset "error" / (0) + ✓ parallel honors stop-on flags with worker recycling with dataset "error" / (1) + ✓ parallel honors stop-on flags with worker recycling with dataset "risky" / (0) + ✓ parallel honors stop-on flags with worker recycling with dataset "risky" / (1) + ✓ parallel honors stop-on flags with worker recycling with dataset "incomplete" / (0) + ✓ parallel honors stop-on flags with worker recycling with dataset "incomplete" / (1) + ✓ parallel honors stop-on flags with worker recycling with dataset "warning" / (0) + ✓ parallel honors stop-on flags with worker recycling with dataset "warning" / (1) + ✓ parallel honors stop-on flags with worker recycling with dataset "notice" / (0) + ✓ parallel honors stop-on flags with worker recycling with dataset "notice" / (1) + ✓ parallel honors stop-on flags with worker recycling with dataset "deprecation" / (0) + ✓ parallel honors stop-on flags with worker recycling with dataset "deprecation" / (1) + ✓ parallel honors stop-on flags with worker recycling with dataset "skipped" / (0) + ✓ parallel honors stop-on flags with worker recycling with dataset "skipped" / (1) + ✓ parallel honors numeric stop-on thresholds with dataset "failure" / (0) + ✓ parallel honors numeric stop-on thresholds with dataset "failure" / (1) + ✓ parallel honors numeric stop-on thresholds with dataset "defect" / (0) + ✓ parallel honors numeric stop-on thresholds with dataset "defect" / (1) + ✓ parallel honors numeric stop-on thresholds with dataset "error" / (0) + ✓ parallel honors numeric stop-on thresholds with dataset "error" / (1) + ✓ parallel honors numeric stop-on thresholds with dataset "risky" / (0) + ✓ parallel honors numeric stop-on thresholds with dataset "risky" / (1) + ✓ parallel honors numeric stop-on thresholds with dataset "incomplete" / (0) + ✓ parallel honors numeric stop-on thresholds with dataset "incomplete" / (1) + ✓ parallel honors numeric stop-on thresholds with dataset "warning" / (0) + ✓ parallel honors numeric stop-on thresholds with dataset "warning" / (1) + ✓ parallel honors numeric stop-on thresholds with dataset "notice" / (0) + ✓ parallel honors numeric stop-on thresholds with dataset "notice" / (1) + ✓ parallel honors numeric stop-on thresholds with dataset "deprecation" / (0) + ✓ parallel honors numeric stop-on thresholds with dataset "deprecation" / (1) + ✓ parallel honors numeric stop-on thresholds with dataset "skipped" / (0) + ✓ parallel honors numeric stop-on thresholds with dataset "skipped" / (1) + ✓ parallel stop-on-defect includes errors warnings and risky tests with ('errors', 'failed', 2) + ✓ parallel stop-on-defect includes errors warnings and risky tests with ('warns', 'warning', 0) + ✓ parallel stop-on-defect includes errors warnings and risky tests with ('is risky', 'risky', 1) + ✓ parallel stop-on-defect allows incomplete and skipped tests with ('is incomplete', 'incomplete') + ✓ parallel stop-on-defect allows incomplete and skipped tests with ('is skipped', 'skipped') + ✓ parallel does not count the same failure twice with (0) + ✓ parallel does not count the same failure twice with (1) + ✓ parallel counts issues hidden by the progress output with ('--stop-on-warning', 'has combined issues', 'failed', …) + ✓ parallel counts issues hidden by the progress output with ('--stop-on-defect=2', 'has combined issues', 'failed', …) + ✓ parallel counts issues hidden by the progress output with ('--stop-on-warning=2', 'has repeated warnings', 'warning', …) + ✓ parallel counts issues hidden by the progress output with ('--stop-on-notice=2', 'has repeated notifications', 'notice', …) + ✓ parallel stops only on the requested deprecation message with ('user deprecation', 1) + ✓ parallel stops only on the requested deprecation message with ('another deprecation', 2) + ✓ parallel honors stop-on flags from XML + ✓ parallel CLI thresholds override XML stop-on flags + ✓ parallel clamps empty and non-positive thresholds like PHPUnit with ('') + ✓ parallel clamps empty and non-positive thresholds like PHPUnit with ('0') + ✓ parallel clamps empty and non-positive thresholds like PHPUnit with ('-2') + ✓ parallel counts a shared issue only once across recycled workers with ('warning', 'shares warning', 'warning') + ✓ parallel counts a shared issue only once across recycled workers with ('defect', 'shares warning', 'warning') + ✓ parallel counts a shared issue only once across recycled workers with ('notice', 'shares notice', 'notice') + ✓ parallel combines stop-on counts from different workers and cleans up their files + PASS Tests\Visual\SingleTestOrDirectory ✓ allows to run a single test ✓ allows to run a directory @@ -2340,4 +2391,4 @@ ✓ pass with dataset with ('my-datas-set-value') ✓ within describe → pass with dataset with ('my-datas-set-value') - Tests: 2 deprecated, 4 warnings, 5 incomplete, 2 notices, 40 todos, 37 skipped, 1667 passed (3670 assertions) \ No newline at end of file + Tests: 2 deprecated, 4 warnings, 5 incomplete, 2 notices, 40 todos, 37 skipped, 1716 passed (3971 assertions) \ No newline at end of file diff --git a/tests/.tests/ParallelStopOn/HasFailureTest.php b/tests/.tests/ParallelStopOn/HasFailureTest.php deleted file mode 100644 index 99aeb933e..000000000 --- a/tests/.tests/ParallelStopOn/HasFailureTest.php +++ /dev/null @@ -1,3 +0,0 @@ -assertTrue(false); diff --git a/tests/Fixtures/Suites/ParallelStopOn.xml b/tests/Fixtures/Suites/ParallelStopOn.xml new file mode 100644 index 000000000..f3af5d4ff --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn.xml @@ -0,0 +1,8 @@ + + + + + ParallelStopOn + + + diff --git a/tests/.tests/ParallelStopOn/HasDeprecationTest.php b/tests/Fixtures/Suites/ParallelStopOn/HasDeprecationTest.php similarity index 100% rename from tests/.tests/ParallelStopOn/HasDeprecationTest.php rename to tests/Fixtures/Suites/ParallelStopOn/HasDeprecationTest.php diff --git a/tests/.tests/ParallelStopOn/HasErrorTest.php b/tests/Fixtures/Suites/ParallelStopOn/HasErrorTest.php similarity index 100% rename from tests/.tests/ParallelStopOn/HasErrorTest.php rename to tests/Fixtures/Suites/ParallelStopOn/HasErrorTest.php diff --git a/tests/Fixtures/Suites/ParallelStopOn/HasFailureTest.php b/tests/Fixtures/Suites/ParallelStopOn/HasFailureTest.php new file mode 100644 index 000000000..27de90e58 --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/HasFailureTest.php @@ -0,0 +1,3 @@ +toBeTrue(); +}); + +it('has repeated warnings', function (): void { + trigger_error('first warning', E_USER_WARNING); + trigger_error('second warning', E_USER_WARNING); + + expect(true)->toBeTrue(); +}); + +it('has repeated notifications', function (): void { + trigger_error('first notice', E_USER_NOTICE); + trigger_error('second notice', E_USER_NOTICE); + + expect(true)->toBeTrue(); +}); diff --git a/tests/Fixtures/Suites/ParallelStopOn/More/HasDeprecationTest.php b/tests/Fixtures/Suites/ParallelStopOn/More/HasDeprecationTest.php new file mode 100644 index 000000000..47e031b06 --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/More/HasDeprecationTest.php @@ -0,0 +1,7 @@ +toBeTrue(); +}); diff --git a/tests/Fixtures/Suites/ParallelStopOn/More/HasErrorTest.php b/tests/Fixtures/Suites/ParallelStopOn/More/HasErrorTest.php new file mode 100644 index 000000000..7c9d0ed5d --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/More/HasErrorTest.php @@ -0,0 +1,5 @@ +markTestIncomplete('work in progress'); +}); diff --git a/tests/Fixtures/Suites/ParallelStopOn/More/HasNoticeTest.php b/tests/Fixtures/Suites/ParallelStopOn/More/HasNoticeTest.php new file mode 100644 index 000000000..552f11b23 --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/More/HasNoticeTest.php @@ -0,0 +1,7 @@ +toBeTrue(); +}); diff --git a/tests/Fixtures/Suites/ParallelStopOn/More/HasRiskyTest.php b/tests/Fixtures/Suites/ParallelStopOn/More/HasRiskyTest.php new file mode 100644 index 000000000..0d2ef4968 --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/More/HasRiskyTest.php @@ -0,0 +1,5 @@ +skip('skipping'); diff --git a/tests/Fixtures/Suites/ParallelStopOn/More/HasWarningTest.php b/tests/Fixtures/Suites/ParallelStopOn/More/HasWarningTest.php new file mode 100644 index 000000000..8fae1020d --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/More/HasWarningTest.php @@ -0,0 +1,7 @@ +toBeTrue(); +}); diff --git a/tests/.tests/ParallelStopOn/Passing/Passing01Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing01Test.php similarity index 100% rename from tests/.tests/ParallelStopOn/Passing/Passing01Test.php rename to tests/Fixtures/Suites/ParallelStopOn/Passing/Passing01Test.php diff --git a/tests/.tests/ParallelStopOn/Passing/Passing02Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing02Test.php similarity index 100% rename from tests/.tests/ParallelStopOn/Passing/Passing02Test.php rename to tests/Fixtures/Suites/ParallelStopOn/Passing/Passing02Test.php diff --git a/tests/.tests/ParallelStopOn/Passing/Passing03Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing03Test.php similarity index 100% rename from tests/.tests/ParallelStopOn/Passing/Passing03Test.php rename to tests/Fixtures/Suites/ParallelStopOn/Passing/Passing03Test.php diff --git a/tests/.tests/ParallelStopOn/Passing/Passing04Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing04Test.php similarity index 100% rename from tests/.tests/ParallelStopOn/Passing/Passing04Test.php rename to tests/Fixtures/Suites/ParallelStopOn/Passing/Passing04Test.php diff --git a/tests/.tests/ParallelStopOn/Passing/Passing05Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing05Test.php similarity index 100% rename from tests/.tests/ParallelStopOn/Passing/Passing05Test.php rename to tests/Fixtures/Suites/ParallelStopOn/Passing/Passing05Test.php diff --git a/tests/.tests/ParallelStopOn/Passing/Passing06Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing06Test.php similarity index 100% rename from tests/.tests/ParallelStopOn/Passing/Passing06Test.php rename to tests/Fixtures/Suites/ParallelStopOn/Passing/Passing06Test.php diff --git a/tests/.tests/ParallelStopOn/Passing/Passing07Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing07Test.php similarity index 100% rename from tests/.tests/ParallelStopOn/Passing/Passing07Test.php rename to tests/Fixtures/Suites/ParallelStopOn/Passing/Passing07Test.php diff --git a/tests/.tests/ParallelStopOn/Passing/Passing08Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing08Test.php similarity index 100% rename from tests/.tests/ParallelStopOn/Passing/Passing08Test.php rename to tests/Fixtures/Suites/ParallelStopOn/Passing/Passing08Test.php diff --git a/tests/.tests/ParallelStopOn/Passing/Passing09Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing09Test.php similarity index 100% rename from tests/.tests/ParallelStopOn/Passing/Passing09Test.php rename to tests/Fixtures/Suites/ParallelStopOn/Passing/Passing09Test.php diff --git a/tests/.tests/ParallelStopOn/Passing/Passing10Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing10Test.php similarity index 100% rename from tests/.tests/ParallelStopOn/Passing/Passing10Test.php rename to tests/Fixtures/Suites/ParallelStopOn/Passing/Passing10Test.php diff --git a/tests/Fixtures/Suites/ParallelStopOnFailure.php b/tests/Fixtures/Suites/ParallelStopOnFailure.php new file mode 100644 index 000000000..71ab55bdb --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOnFailure.php @@ -0,0 +1,20 @@ += $deadline) { + throw new RuntimeException('The second parallel worker did not reach the barrier.'); + } + + usleep(1000); + } + } + + expect(false)->toBeTrue(); +}; diff --git a/tests/Fixtures/Suites/ParallelStopOnSharedIssue.php b/tests/Fixtures/Suites/ParallelStopOnSharedIssue.php new file mode 100644 index 000000000..ff7dd8ecb --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOnSharedIssue.php @@ -0,0 +1,7 @@ + function () use ($severity): void { + trigger_error('shared issue', $severity); + + expect(true)->toBeTrue(); +}; diff --git a/tests/Visual/Parallel.php b/tests/Visual/Parallel.php index a73e47e76..3b5919264 100644 --- a/tests/Visual/Parallel.php +++ b/tests/Visual/Parallel.php @@ -38,36 +38,6 @@ ->toContain('Parallel: 3 processes'); })->skipOnWindows(); -test('parallel stop-on-* fixture runs every case without a stop flag', function (string $testName) use ($run): void { - $output = $run('--filter', "($testName|passes)", 'tests/.tests/ParallelStopOn'); - expect($output)->toContain('10 passed'); -})->with([ - 'fails', - 'errors', - 'is risky', - 'is incomplete', - 'warns', - 'notices', - 'deprecates', - 'is skipped', -])->skipOnWindows(); - -test('parallel honors --stop-on-*', function (string $flag, string $testName) use ($run): void { - $output = $run('--filter', "($testName|passes)", $flag, 'tests/.tests/ParallelStopOn'); - preg_match('/(\d+) passed/', $output, $matches); - expect((int) ($matches[1] ?? 0))->toBeLessThan(10); -})->with([ - 'failure' => ['--stop-on-failure', 'fails'], - 'defect' => ['--stop-on-defect', 'fails'], - 'error' => ['--stop-on-error', 'errors'], - 'risky' => ['--stop-on-risky', 'is risky'], - 'incomplete' => ['--stop-on-incomplete', 'is incomplete'], - 'warning' => ['--stop-on-warning', 'warns'], - 'notice' => ['--stop-on-notice', 'notices'], - 'deprecation' => ['--stop-on-deprecation', 'deprecates'], - 'skipped' => ['--stop-on-skipped', 'is skipped'], -])->skipOnWindows(); - test('parallel lets extensions register event subscribers', function (): void { $process = new Process( ['php', 'bin/pest', '--configuration', 'tests/Fixtures/Suites/ParallelExtensionSubscriber.xml', '--parallel', '--processes=2'], diff --git a/tests/Visual/ParallelStopOn.php b/tests/Visual/ParallelStopOn.php new file mode 100644 index 000000000..7516f4993 --- /dev/null +++ b/tests/Visual/ParallelStopOn.php @@ -0,0 +1,164 @@ + 'DefaultPrinter', 'COLLISION_IGNORE_DURATION' => 'true', 'PAO_DISABLE' => '1', ...$environment], + ); + + $process->run(); + + $output = (string) removeAnsiEscapeSequences($process->getOutput()); + + expect($process->getExitCode())->toBe($exitCode, $process->getErrorOutput().$output) + ->and($process->getErrorOutput())->toBeEmpty() + ->and($output)->toMatch('/Tests:\s+\d+ [^\r\n]+\(\d+ assertions\)/'); + + return $output; +}; + +$outcomes = [ + 'failure' => ['failure', 'fails', 'failed', 1], + 'defect' => ['defect', 'fails', 'failed', 1], + 'error' => ['error', 'errors', 'failed', 2], + 'risky' => ['risky', 'is risky', 'risky', 1], + 'incomplete' => ['incomplete', 'is incomplete', 'incomplete', 0], + 'warning' => ['warning', 'warns', 'warning', 0], + 'notice' => ['notice', 'notices', 'notice', 0], + 'deprecation' => ['deprecation', 'deprecates', 'deprecated', 1], + 'skipped' => ['skipped', 'is skipped', 'skipped', 0], +]; + +test('parallel stop-on fixtures run completely without a stop flag', function (string $flag, string $name, string $outcome, int $exitCode) use ($run): void { + $output = $run($exitCode, ['--filter', "($name|passes)"]); + $plural = in_array($outcome, ['warning', 'notice'], true) ? $outcome.'s' : $outcome; + + expect($output)->toContain("Tests: 2 $plural, 10 passed"); +})->with(array_diff_key($outcomes, ['defect' => true]))->skipOnWindows(); + +test('parallel honors stop-on flags with worker recycling', function (string $flag, string $name, string $outcome, int $exitCode, int $batchSize) use ($run): void { + $output = $run($exitCode, ['--filter', "($name|passes)", "--stop-on-$flag", "--max-batch-size=$batchSize"]); + + expect($output)->toContain("Tests: 1 $outcome (") + ->not->toMatch('/\d+ passed/'); +})->with($outcomes)->with([0, 1])->skipOnWindows(); + +test('parallel honors numeric stop-on thresholds', function (string $flag, string $name, string $outcome, int $exitCode, int $batchSize) use ($run): void { + $output = $run($exitCode, ['--filter', "($name|passes)", "--stop-on-$flag=2", "--max-batch-size=$batchSize"]); + $plural = in_array($outcome, ['warning', 'notice'], true) ? $outcome.'s' : $outcome; + + expect($output)->toContain("Tests: 2 $plural (") + ->not->toMatch('/\d+ passed/'); +})->with($outcomes)->with([0, 1])->skipOnWindows(); + +test('parallel stop-on-defect includes errors warnings and risky tests', function (string $name, string $outcome, int $exitCode) use ($run): void { + $output = $run($exitCode, ['--filter', "($name|passes)", '--stop-on-defect', '--max-batch-size=1']); + + expect($output)->toContain("Tests: 1 $outcome (") + ->not->toMatch('/\d+ passed/'); +})->with([ + ['errors', 'failed', 2], + ['warns', 'warning', 0], + ['is risky', 'risky', 1], +])->skipOnWindows(); + +test('parallel stop-on-defect allows incomplete and skipped tests', function (string $name, string $outcome) use ($run): void { + $output = $run(0, ['--filter', "($name|passes)", '--stop-on-defect']); + + expect($output)->toContain("Tests: 2 $outcome, 10 passed"); +})->with([ + ['is incomplete', 'incomplete'], + ['is skipped', 'skipped'], +])->skipOnWindows(); + +test('parallel does not count the same failure twice', function (int $batchSize) use ($run): void { + $output = $run(1, ['--filter', '(fails|passes)', '--stop-on-failure=3', "--max-batch-size=$batchSize"]); + + expect($output)->toContain('Tests: 2 failed, 10 passed'); +})->with([0, 1])->skipOnWindows(); + +test('parallel counts issues hidden by the progress output', function (string $flag, string $name, string $outcome, int $exitCode) use ($run): void { + $output = $run($exitCode, ['--filter', "($name|passes)", $flag, '--max-batch-size=1']); + + expect($output)->toContain("Tests: 1 $outcome (") + ->not->toMatch('/\d+ passed/'); +})->with([ + ['--stop-on-warning', 'has combined issues', 'failed', 1], + ['--stop-on-defect=2', 'has combined issues', 'failed', 1], + ['--stop-on-warning=2', 'has repeated warnings', 'warning', 0], + ['--stop-on-notice=2', 'has repeated notifications', 'notice', 0], +])->skipOnWindows(); + +test('parallel stops only on the requested deprecation message', function (string $message, int $count) use ($run): void { + $output = $run(1, ['--filter', '(deprecates|passes)', "--stop-on-deprecation=$message", '--max-batch-size=1']); + + expect($output)->toContain("Tests: $count deprecated"); + + if ($count === 1) { + expect($output)->not->toMatch('/\d+ passed/'); + } else { + expect($output)->toContain('10 passed'); + } +})->with([ + ['user deprecation', 1], + ['another deprecation', 2], +])->skipOnWindows(); + +test('parallel honors stop-on flags from XML', function () use ($run): void { + $output = $run(1, ['--configuration=tests/Fixtures/Suites/ParallelStopOn.xml', '--filter', '(fails|passes)', '--max-batch-size=1']); + + expect($output)->toContain('Tests: 1 failed (') + ->not->toMatch('/\d+ passed/'); +})->skipOnWindows(); + +test('parallel CLI thresholds override XML stop-on flags', function () use ($run): void { + $output = $run(1, ['--configuration=tests/Fixtures/Suites/ParallelStopOn.xml', '--filter', '(fails|passes)', '--stop-on-failure=2']); + + expect($output)->toContain('Tests: 2 failed (') + ->not->toMatch('/\d+ passed/'); +})->skipOnWindows(); + +test('parallel clamps empty and non-positive thresholds like PHPUnit', function (string $threshold) use ($run): void { + $output = $run(1, ['--filter', '(fails|passes)', "--stop-on-failure=$threshold"]); + + expect($output)->toContain('Tests: 1 failed (') + ->not->toMatch('/\d+ passed/'); +})->with(['', '0', '-2'])->skipOnWindows(); + +test('parallel counts a shared issue only once across recycled workers', function (string $flag, string $name, string $outcome) use ($run): void { + $output = $run(0, ['--filter', "($name|passes)", "--stop-on-$flag=2", '--max-batch-size=1']); + + expect($output)->toMatch('/Tests:\s+\d+ '.$outcome.'s?, 10 passed/'); +})->with([ + ['warning', 'shares warning', 'warning'], + ['defect', 'shares warning', 'warning'], + ['notice', 'shares notice', 'notice'], +])->skipOnWindows(); + +test('parallel combines stop-on counts from different workers and cleans up their files', function () use ($run): void { + $directory = sys_get_temp_dir().'/pest-stop-on-'.bin2hex(random_bytes(8)); + mkdir($directory); + + try { + $output = $run( + 1, + ['--processes=2', '--stop-on-failure=2', '--filter', '(fails|passes)', '--tmp-dir='.$directory], + ['PEST_TEST_STOP_BARRIER' => $directory], + ); + + expect($output)->toContain('Tests: 2 failed') + ->not->toContain('10 passed') + ->not->toContain('did not reach the barrier') + ->and(glob($directory.'/worker-*'))->toHaveCount(2) + ->and(glob($directory.'/paratest_*'))->toBeEmpty(); + } finally { + foreach (glob($directory.'/worker-*') as $file) { + unlink($file); + } + + rmdir($directory); + } +})->skipOnWindows();