diff --git a/bin/worker.php b/bin/worker.php index 8a4574dbd..c4bce8562 100644 --- a/bin/worker.php +++ b/bin/worker.php @@ -6,8 +6,10 @@ use ParaTest\WrapperRunner\WrapperWorker; use Pest\Kernel; use Pest\Plugins\Actions\CallsHandleArguments; +use Pest\Plugins\Parallel\Paratest\StopConditions; use Pest\Support\Container; use Pest\TestSuite; +use PHPUnit\TextUI\Configuration\Registry as ConfigurationRegistry; use Symfony\Component\Console\Input\ArgvInput; use Symfony\Component\Console\Output\ConsoleOutput; use Symfony\Component\Console\Output\OutputInterface; @@ -99,6 +101,8 @@ (int) ($getopt['testdox-columns'] ?? null), ); + $stopConditions = null; + while (true) { if (feof(STDIN)) { $application->end(); @@ -113,6 +117,9 @@ $exitCode = $application->runTest(realpath(trim($testPath))); + $stopConditions ??= new StopConditions(ConfigurationRegistry::get()); + $stopConditions->write($getopt['test-result-file'].'.stop'); + fwrite($statusFile, (string) $exitCode); fflush($statusFile); } diff --git a/src/Plugins/Parallel.php b/src/Plugins/Parallel.php index cd9ecab5d..b2a5d1fb7 100644 --- a/src/Plugins/Parallel.php +++ b/src/Plugins/Parallel.php @@ -29,6 +29,7 @@ final class Parallel implements HandlesArguments Parallel\Handlers\Parallel::class, Parallel\Handlers\Pest::class, Parallel\Handlers\Laravel::class, + Parallel\Handlers\StopOn::class, ]; /** @@ -148,6 +149,8 @@ private function paratestCommand(): Application $command->setName('Pest'); $command->setVersion(version()); + Parallel\Handlers\StopOn::configureOptions($command->find('paratest')->getDefinition()); + return $command; } diff --git a/src/Plugins/Parallel/Handlers/StopOn.php b/src/Plugins/Parallel/Handlers/StopOn.php new file mode 100644 index 000000000..cc06e2f0d --- /dev/null +++ b/src/Plugins/Parallel/Handlers/StopOn.php @@ -0,0 +1,76 @@ + $argument) { + [$option, $value] = array_pad(explode('=', $argument, 2), 2, ''); + + if (in_array($option, self::OPTIONS, true) + && in_array($value, ['', '0'], true)) { + $arguments[$index] = $option.'=1'; + } + } + + return $arguments; + } + + public function handleWorkerArguments(array $arguments): array + { + $arguments = array_values($arguments); + + foreach ($arguments as $index => $argument) { + if (! in_array($argument, self::OPTIONS, true) + || ! isset($arguments[$index + 1]) + || str_starts_with($arguments[$index + 1], '--')) { + continue; + } + + $arguments[$index] .= '='.$arguments[$index + 1]; + unset($arguments[$index + 1]); + } + + return array_values($arguments); + } + + public static function configureOptions(InputDefinition $definition): void + { + $options = $definition->getOptions(); + + foreach (self::OPTIONS as $argument) { + $name = substr($argument, 2); + $option = $options[$name]; + $options[$name] = new InputOption( + $name, + $option->getShortcut(), + InputOption::VALUE_OPTIONAL, + $option->getDescription(), + false, + ); + } + + $definition->setOptions($options); + } +} diff --git a/src/Plugins/Parallel/Paratest/StopConditions.php b/src/Plugins/Parallel/Paratest/StopConditions.php new file mode 100644 index 000000000..f5a197515 --- /dev/null +++ b/src/Plugins/Parallel/Paratest/StopConditions.php @@ -0,0 +1,216 @@ + + */ + private readonly array $thresholds; + + /** + * @var array> + */ + private array $workerCounts = []; + + /** + * @var array + */ + private array $totals = []; + + /** + * @var array> + */ + private array $issueIds = []; + + public function __construct(private readonly Configuration $configuration) + { + $this->thresholds = array_filter([ + 'defect' => $configuration->stopOnDefectThreshold(), + 'error' => $configuration->stopOnErrorThreshold(), + 'failure' => $configuration->stopOnFailureThreshold(), + 'warning' => $configuration->stopOnWarningThreshold(), + 'risky' => $configuration->stopOnRiskyThreshold(), + 'deprecation' => $configuration->stopOnDeprecationThreshold(), + 'notice' => $configuration->stopOnNoticeThreshold(), + 'incomplete' => $configuration->stopOnIncompleteThreshold(), + 'skipped' => $configuration->stopOnSkippedThreshold(), + ], static fn (int $threshold): bool => $threshold > 0); + } + + public function write(string $filename): void + { + if ($this->thresholds === []) { + return; + } + + $result = TestResultFacade::result(); + $state = json_encode([ + 'counts' => $this->counts($result), + 'issues' => $this->issues($result), + ], JSON_THROW_ON_ERROR); + + if (file_put_contents($filename.'.tmp', $state) === false + || ! rename($filename.'.tmp', $filename)) { + throw new RuntimeException(sprintf('Unable to write the parallel stop counts to [%s].', $filename)); + } + } + + public function read(string $filename): void + { + if ($this->thresholds === []) { + return; + } + + $contents = file_get_contents($filename); + + if ($contents === false) { + throw new RuntimeException(sprintf('Unable to read the parallel stop counts from [%s].', $filename)); + } + + $state = json_decode($contents, true, flags: JSON_THROW_ON_ERROR); + + if (! is_array($state) + || ! isset($state['counts'], $state['issues']) + || ! is_array($state['counts']) + || ! is_array($state['issues'])) { + throw new RuntimeException(sprintf('Invalid parallel stop counts in [%s].', $filename)); + } + + $this->addCounts($filename, $state['counts']); + $this->addIssues($filename, $state['issues']); + } + + public function shouldStop(): bool + { + return array_any($this->thresholds, function (int $threshold, string $outcome): bool { + $issueType = $outcome === 'defect' ? 'warning' : $outcome; + $count = ($this->totals[$outcome] ?? 0) + count($this->issueIds[$issueType] ?? []); + + return $count >= $threshold; + }); + } + + /** + * @return array + */ + private function counts(TestResult $result): array + { + $counts = [ + 'error' => $result->numberOfTestErroredEvents(), + 'failure' => $result->numberOfTestFailedEvents(), + 'warning' => $result->numberOfPhpunitWarnings(), + 'risky' => $result->numberOfTestsWithTestConsideredRiskyEvents(), + 'deprecation' => $this->numberOfDeprecations(), + 'notice' => 0, + 'incomplete' => $result->numberOfTestMarkedIncompleteEvents(), + 'skipped' => $result->numberOfTestSkippedEvents(), + ]; + $counts['defect'] = $counts['error'] + $counts['failure'] + $counts['warning'] + $counts['risky']; + + return $counts; + } + + /** + * @return array<'warning'|'notice', list> + */ + private function issues(TestResult $result): array + { + $issues = []; + + if (isset($this->thresholds['warning']) || isset($this->thresholds['defect'])) { + $issues['warning'] = array_merge( + $this->issueKeys($result->warnings(), 'user'), + $this->issueKeys($result->phpWarnings(), 'php'), + ); + } + + if (isset($this->thresholds['notice'])) { + $issues['notice'] = array_merge( + $this->issueKeys($result->notices(), 'user'), + $this->issueKeys($result->phpNotices(), 'php'), + ); + } + + return $issues; + } + + /** + * @return non-negative-int + */ + private function numberOfDeprecations(): int + { + $deprecations = DeprecationCollector::filteredDeprecations(); + + if ($this->configuration->hasSpecificDeprecationToStopOn()) { + $message = $this->configuration->specificDeprecationToStopOn(); + $deprecations = array_filter($deprecations, static fn (string $deprecation): bool => str_contains($deprecation, $message)); + } + + return count($deprecations); + } + + /** + * @param array $counts + */ + private function addCounts(string $filename, array $counts): void + { + foreach (array_keys($this->thresholds) as $outcome) { + if (! isset($counts[$outcome]) || ! is_int($counts[$outcome]) || $counts[$outcome] < 0) { + throw new RuntimeException(sprintf('Invalid parallel stop counts in [%s].', $filename)); + } + + $previous = $this->workerCounts[$filename][$outcome] ?? 0; + $increase = $counts[$outcome] - $previous; + + if ($increase < 0) { + throw new RuntimeException(sprintf('Parallel stop counts decreased in [%s].', $filename)); + } + + $this->totals[$outcome] = ($this->totals[$outcome] ?? 0) + $increase; + $this->workerCounts[$filename][$outcome] = $counts[$outcome]; + } + } + + /** + * @param array $issues + */ + private function addIssues(string $filename, array $issues): void + { + foreach ($issues as $outcome => $ids) { + if (! in_array($outcome, ['warning', 'notice'], true) || ! is_array($ids)) { + throw new RuntimeException(sprintf('Invalid parallel stop issues in [%s].', $filename)); + } + + foreach ($ids as $id) { + if (! is_string($id)) { + throw new RuntimeException(sprintf('Invalid parallel stop issue in [%s].', $filename)); + } + + $this->issueIds[$outcome][$id] = true; + } + } + } + + /** + * @param list $issues + * @return list + */ + private function issueKeys(array $issues, string $origin): array + { + return array_map( + static fn (Issue $issue): string => hash('sha256', implode(':', [$origin, $issue->file(), $issue->line(), $issue->description()])), + $issues, + ); + } +} diff --git a/src/Plugins/Parallel/Paratest/WrapperRunner.php b/src/Plugins/Parallel/Paratest/WrapperRunner.php index 0b54e20cd..676af83f3 100644 --- a/src/Plugins/Parallel/Paratest/WrapperRunner.php +++ b/src/Plugins/Parallel/Paratest/WrapperRunner.php @@ -78,8 +78,6 @@ final class WrapperRunner implements RunnerInterface /** @var list */ private array $pending = []; - private int $exitcode = -1; - /** @var array */ private array $workers = []; @@ -122,6 +120,8 @@ final class WrapperRunner implements RunnerInterface /** @var non-empty-string[] */ private readonly array $parameters; + private readonly StopConditions $stopConditions; + private CodeCoverageFilterRegistry $codeCoverageFilterRegistry; /** @var non-empty-string */ @@ -133,6 +133,7 @@ public function __construct( ) { $this->printer = new ResultPrinter($output, $options); $this->timer = new Timer; + $this->stopConditions = new StopConditions($options->configuration); $wrapper = realpath( dirname(__DIR__, 4).DIRECTORY_SEPARATOR.'bin'.DIRECTORY_SEPARATOR.'worker.php', @@ -248,17 +249,18 @@ private function assignAllPendingTests(): void $this->flushWorker($worker); + if ($this->stopConditions->shouldStop()) { + $this->pending = []; + + break; + } + if ($batchSize !== 0 && $this->batches[$token] === $batchSize) { $this->destroyWorker($token); $worker = $this->startWorker($token); } - if ( - $this->exitcode > 0 - && $this->options->configuration->stopOnFailureThreshold() > 0 - ) { - $this->pending = []; - } elseif (($pending = array_shift($this->pending)) !== null) { + if (($pending = array_shift($this->pending)) !== null) { $worker->assign($pending); $this->batches[$token]++; } @@ -275,6 +277,7 @@ private function flushWorker(WrapperWorker $worker): void if ($testResultFile !== '') { $this->requiredTestResultFiles[$testResultFile] = true; + $this->stopConditions->read($testResultFile.'.stop'); } if (isset($worker->coverageFile) && $worker->coverageFile->getPathname() !== '') { @@ -282,7 +285,6 @@ private function flushWorker(WrapperWorker $worker): void } } - $this->exitcode = max($this->exitcode, $worker->getExitCode()); $this->printer->printFeedback( $worker->progressFile, $worker->unexpectedOutputFile, @@ -501,7 +503,11 @@ private function complete(TestResult $testResultSum): int $this->clearFiles($this->statusFiles); $this->clearFiles($this->progressFiles); $this->clearFiles($this->unexpectedOutputFiles); - $this->clearFiles($this->testResultFiles); + + foreach ($this->testResultFiles as $file) { + $this->clearFiles([$file, new SplFileInfo($file->getPathname().'.stop')]); + } + $this->clearFiles($this->resultCacheFiles); $this->clearFiles($this->coverageFiles); $this->clearFiles($this->junitFiles); diff --git a/tests/.snapshots/success.txt b/tests/.snapshots/success.txt index de896b1d3..eca0d4843 100644 --- a/tests/.snapshots/success.txt +++ b/tests/.snapshots/success.txt @@ -2294,6 +2294,74 @@ PASS Tests\Visual\ParallelNestedDatasets ✓ parallel loads nested datasets from nested directories + PASS Tests\Visual\ParallelStopOn + ✓ parallel stop-on fixtures run completely without a stop flag with dataset "failure" + ✓ parallel stop-on fixtures run completely without a stop flag with dataset "error" + ✓ parallel stop-on fixtures run completely without a stop flag with dataset "risky" + ✓ parallel stop-on fixtures run completely without a stop flag with dataset "incomplete" + ✓ parallel stop-on fixtures run completely without a stop flag with dataset "warning" + ✓ parallel stop-on fixtures run completely without a stop flag with dataset "notice" + ✓ parallel stop-on fixtures run completely without a stop flag with dataset "deprecation" + ✓ parallel stop-on fixtures run completely without a stop flag with dataset "skipped" + ✓ parallel honors stop-on flags with worker recycling with dataset "failure" / (0) + ✓ parallel honors stop-on flags with worker recycling with dataset "failure" / (1) + ✓ parallel honors stop-on flags with worker recycling with dataset "defect" / (0) + ✓ parallel honors stop-on flags with worker recycling with dataset "defect" / (1) + ✓ parallel honors stop-on flags with worker recycling with dataset "error" / (0) + ✓ parallel honors stop-on flags with worker recycling with dataset "error" / (1) + ✓ parallel honors stop-on flags with worker recycling with dataset "risky" / (0) + ✓ parallel honors stop-on flags with worker recycling with dataset "risky" / (1) + ✓ parallel honors stop-on flags with worker recycling with dataset "incomplete" / (0) + ✓ parallel honors stop-on flags with worker recycling with dataset "incomplete" / (1) + ✓ parallel honors stop-on flags with worker recycling with dataset "warning" / (0) + ✓ parallel honors stop-on flags with worker recycling with dataset "warning" / (1) + ✓ parallel honors stop-on flags with worker recycling with dataset "notice" / (0) + ✓ parallel honors stop-on flags with worker recycling with dataset "notice" / (1) + ✓ parallel honors stop-on flags with worker recycling with dataset "deprecation" / (0) + ✓ parallel honors stop-on flags with worker recycling with dataset "deprecation" / (1) + ✓ parallel honors stop-on flags with worker recycling with dataset "skipped" / (0) + ✓ parallel honors stop-on flags with worker recycling with dataset "skipped" / (1) + ✓ parallel honors numeric stop-on thresholds with dataset "failure" / (0) + ✓ parallel honors numeric stop-on thresholds with dataset "failure" / (1) + ✓ parallel honors numeric stop-on thresholds with dataset "defect" / (0) + ✓ parallel honors numeric stop-on thresholds with dataset "defect" / (1) + ✓ parallel honors numeric stop-on thresholds with dataset "error" / (0) + ✓ parallel honors numeric stop-on thresholds with dataset "error" / (1) + ✓ parallel honors numeric stop-on thresholds with dataset "risky" / (0) + ✓ parallel honors numeric stop-on thresholds with dataset "risky" / (1) + ✓ parallel honors numeric stop-on thresholds with dataset "incomplete" / (0) + ✓ parallel honors numeric stop-on thresholds with dataset "incomplete" / (1) + ✓ parallel honors numeric stop-on thresholds with dataset "warning" / (0) + ✓ parallel honors numeric stop-on thresholds with dataset "warning" / (1) + ✓ parallel honors numeric stop-on thresholds with dataset "notice" / (0) + ✓ parallel honors numeric stop-on thresholds with dataset "notice" / (1) + ✓ parallel honors numeric stop-on thresholds with dataset "deprecation" / (0) + ✓ parallel honors numeric stop-on thresholds with dataset "deprecation" / (1) + ✓ parallel honors numeric stop-on thresholds with dataset "skipped" / (0) + ✓ parallel honors numeric stop-on thresholds with dataset "skipped" / (1) + ✓ parallel stop-on-defect includes errors warnings and risky tests with ('errors', 'failed', 2) + ✓ parallel stop-on-defect includes errors warnings and risky tests with ('warns', 'warning', 0) + ✓ parallel stop-on-defect includes errors warnings and risky tests with ('is risky', 'risky', 1) + ✓ parallel stop-on-defect allows incomplete and skipped tests with ('is incomplete', 'incomplete') + ✓ parallel stop-on-defect allows incomplete and skipped tests with ('is skipped', 'skipped') + ✓ parallel does not count the same failure twice with (0) + ✓ parallel does not count the same failure twice with (1) + ✓ parallel counts issues hidden by the progress output with ('--stop-on-warning', 'has combined issues', 'failed', …) + ✓ parallel counts issues hidden by the progress output with ('--stop-on-defect=2', 'has combined issues', 'failed', …) + ✓ parallel counts issues hidden by the progress output with ('--stop-on-warning=2', 'has repeated warnings', 'warning', …) + ✓ parallel counts issues hidden by the progress output with ('--stop-on-notice=2', 'has repeated notifications', 'notice', …) + ✓ parallel stops only on the requested deprecation message with ('user deprecation', 1) + ✓ parallel stops only on the requested deprecation message with ('another deprecation', 2) + ✓ parallel honors stop-on flags from XML + ✓ parallel CLI thresholds override XML stop-on flags + ✓ parallel clamps empty and non-positive thresholds like PHPUnit with ('') + ✓ parallel clamps empty and non-positive thresholds like PHPUnit with ('0') + ✓ parallel clamps empty and non-positive thresholds like PHPUnit with ('-2') + ✓ parallel counts a shared issue only once across recycled workers with ('warning', 'shares warning', 'warning') + ✓ parallel counts a shared issue only once across recycled workers with ('defect', 'shares warning', 'warning') + ✓ parallel counts a shared issue only once across recycled workers with ('notice', 'shares notice', 'notice') + ✓ parallel combines stop-on counts from different workers and cleans up their files + PASS Tests\Visual\SingleTestOrDirectory ✓ allows to run a single test ✓ allows to run a directory @@ -2323,4 +2391,4 @@ ✓ pass with dataset with ('my-datas-set-value') ✓ within describe → pass with dataset with ('my-datas-set-value') - Tests: 2 deprecated, 4 warnings, 5 incomplete, 2 notices, 40 todos, 37 skipped, 1650 passed (3653 assertions) \ No newline at end of file + Tests: 2 deprecated, 4 warnings, 5 incomplete, 2 notices, 40 todos, 37 skipped, 1716 passed (3971 assertions) \ No newline at end of file diff --git a/tests/Fixtures/Suites/ParallelStopOn.xml b/tests/Fixtures/Suites/ParallelStopOn.xml new file mode 100644 index 000000000..f3af5d4ff --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn.xml @@ -0,0 +1,8 @@ + + + + + ParallelStopOn + + + diff --git a/tests/Fixtures/Suites/ParallelStopOn/HasDeprecationTest.php b/tests/Fixtures/Suites/ParallelStopOn/HasDeprecationTest.php new file mode 100644 index 000000000..47e031b06 --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/HasDeprecationTest.php @@ -0,0 +1,7 @@ +toBeTrue(); +}); diff --git a/tests/Fixtures/Suites/ParallelStopOn/HasErrorTest.php b/tests/Fixtures/Suites/ParallelStopOn/HasErrorTest.php new file mode 100644 index 000000000..7c9d0ed5d --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/HasErrorTest.php @@ -0,0 +1,5 @@ +markTestIncomplete('work in progress'); +}); diff --git a/tests/Fixtures/Suites/ParallelStopOn/HasNoticeTest.php b/tests/Fixtures/Suites/ParallelStopOn/HasNoticeTest.php new file mode 100644 index 000000000..552f11b23 --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/HasNoticeTest.php @@ -0,0 +1,7 @@ +toBeTrue(); +}); diff --git a/tests/Fixtures/Suites/ParallelStopOn/HasRiskyTest.php b/tests/Fixtures/Suites/ParallelStopOn/HasRiskyTest.php new file mode 100644 index 000000000..0d2ef4968 --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/HasRiskyTest.php @@ -0,0 +1,5 @@ +skip('skipping'); diff --git a/tests/Fixtures/Suites/ParallelStopOn/HasWarningTest.php b/tests/Fixtures/Suites/ParallelStopOn/HasWarningTest.php new file mode 100644 index 000000000..8fae1020d --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/HasWarningTest.php @@ -0,0 +1,7 @@ +toBeTrue(); +}); diff --git a/tests/Fixtures/Suites/ParallelStopOn/MixedOutcomesTest.php b/tests/Fixtures/Suites/ParallelStopOn/MixedOutcomesTest.php new file mode 100644 index 000000000..8a8532b04 --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/MixedOutcomesTest.php @@ -0,0 +1,21 @@ +toBeTrue(); +}); + +it('has repeated warnings', function (): void { + trigger_error('first warning', E_USER_WARNING); + trigger_error('second warning', E_USER_WARNING); + + expect(true)->toBeTrue(); +}); + +it('has repeated notifications', function (): void { + trigger_error('first notice', E_USER_NOTICE); + trigger_error('second notice', E_USER_NOTICE); + + expect(true)->toBeTrue(); +}); diff --git a/tests/Fixtures/Suites/ParallelStopOn/More/HasDeprecationTest.php b/tests/Fixtures/Suites/ParallelStopOn/More/HasDeprecationTest.php new file mode 100644 index 000000000..47e031b06 --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/More/HasDeprecationTest.php @@ -0,0 +1,7 @@ +toBeTrue(); +}); diff --git a/tests/Fixtures/Suites/ParallelStopOn/More/HasErrorTest.php b/tests/Fixtures/Suites/ParallelStopOn/More/HasErrorTest.php new file mode 100644 index 000000000..7c9d0ed5d --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/More/HasErrorTest.php @@ -0,0 +1,5 @@ +markTestIncomplete('work in progress'); +}); diff --git a/tests/Fixtures/Suites/ParallelStopOn/More/HasNoticeTest.php b/tests/Fixtures/Suites/ParallelStopOn/More/HasNoticeTest.php new file mode 100644 index 000000000..552f11b23 --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/More/HasNoticeTest.php @@ -0,0 +1,7 @@ +toBeTrue(); +}); diff --git a/tests/Fixtures/Suites/ParallelStopOn/More/HasRiskyTest.php b/tests/Fixtures/Suites/ParallelStopOn/More/HasRiskyTest.php new file mode 100644 index 000000000..0d2ef4968 --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/More/HasRiskyTest.php @@ -0,0 +1,5 @@ +skip('skipping'); diff --git a/tests/Fixtures/Suites/ParallelStopOn/More/HasWarningTest.php b/tests/Fixtures/Suites/ParallelStopOn/More/HasWarningTest.php new file mode 100644 index 000000000..8fae1020d --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/More/HasWarningTest.php @@ -0,0 +1,7 @@ +toBeTrue(); +}); diff --git a/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing01Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing01Test.php new file mode 100644 index 000000000..8dc2e1fb5 --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing01Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing02Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing02Test.php new file mode 100644 index 000000000..86045d01e --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing02Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing03Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing03Test.php new file mode 100644 index 000000000..057c6070f --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing03Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing04Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing04Test.php new file mode 100644 index 000000000..883200ba6 --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing04Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing05Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing05Test.php new file mode 100644 index 000000000..8a050a111 --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing05Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing06Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing06Test.php new file mode 100644 index 000000000..ceef2a2fb --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing06Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing07Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing07Test.php new file mode 100644 index 000000000..1f0a6b0f6 --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing07Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing08Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing08Test.php new file mode 100644 index 000000000..bae06622c --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing08Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing09Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing09Test.php new file mode 100644 index 000000000..bcc6b5e14 --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing09Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing10Test.php b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing10Test.php new file mode 100644 index 000000000..edc06d046 --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOn/Passing/Passing10Test.php @@ -0,0 +1,3 @@ +assertTrue(true); diff --git a/tests/Fixtures/Suites/ParallelStopOnFailure.php b/tests/Fixtures/Suites/ParallelStopOnFailure.php new file mode 100644 index 000000000..71ab55bdb --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOnFailure.php @@ -0,0 +1,20 @@ += $deadline) { + throw new RuntimeException('The second parallel worker did not reach the barrier.'); + } + + usleep(1000); + } + } + + expect(false)->toBeTrue(); +}; diff --git a/tests/Fixtures/Suites/ParallelStopOnSharedIssue.php b/tests/Fixtures/Suites/ParallelStopOnSharedIssue.php new file mode 100644 index 000000000..ff7dd8ecb --- /dev/null +++ b/tests/Fixtures/Suites/ParallelStopOnSharedIssue.php @@ -0,0 +1,7 @@ + function () use ($severity): void { + trigger_error('shared issue', $severity); + + expect(true)->toBeTrue(); +}; diff --git a/tests/Visual/ParallelStopOn.php b/tests/Visual/ParallelStopOn.php new file mode 100644 index 000000000..7516f4993 --- /dev/null +++ b/tests/Visual/ParallelStopOn.php @@ -0,0 +1,164 @@ + 'DefaultPrinter', 'COLLISION_IGNORE_DURATION' => 'true', 'PAO_DISABLE' => '1', ...$environment], + ); + + $process->run(); + + $output = (string) removeAnsiEscapeSequences($process->getOutput()); + + expect($process->getExitCode())->toBe($exitCode, $process->getErrorOutput().$output) + ->and($process->getErrorOutput())->toBeEmpty() + ->and($output)->toMatch('/Tests:\s+\d+ [^\r\n]+\(\d+ assertions\)/'); + + return $output; +}; + +$outcomes = [ + 'failure' => ['failure', 'fails', 'failed', 1], + 'defect' => ['defect', 'fails', 'failed', 1], + 'error' => ['error', 'errors', 'failed', 2], + 'risky' => ['risky', 'is risky', 'risky', 1], + 'incomplete' => ['incomplete', 'is incomplete', 'incomplete', 0], + 'warning' => ['warning', 'warns', 'warning', 0], + 'notice' => ['notice', 'notices', 'notice', 0], + 'deprecation' => ['deprecation', 'deprecates', 'deprecated', 1], + 'skipped' => ['skipped', 'is skipped', 'skipped', 0], +]; + +test('parallel stop-on fixtures run completely without a stop flag', function (string $flag, string $name, string $outcome, int $exitCode) use ($run): void { + $output = $run($exitCode, ['--filter', "($name|passes)"]); + $plural = in_array($outcome, ['warning', 'notice'], true) ? $outcome.'s' : $outcome; + + expect($output)->toContain("Tests: 2 $plural, 10 passed"); +})->with(array_diff_key($outcomes, ['defect' => true]))->skipOnWindows(); + +test('parallel honors stop-on flags with worker recycling', function (string $flag, string $name, string $outcome, int $exitCode, int $batchSize) use ($run): void { + $output = $run($exitCode, ['--filter', "($name|passes)", "--stop-on-$flag", "--max-batch-size=$batchSize"]); + + expect($output)->toContain("Tests: 1 $outcome (") + ->not->toMatch('/\d+ passed/'); +})->with($outcomes)->with([0, 1])->skipOnWindows(); + +test('parallel honors numeric stop-on thresholds', function (string $flag, string $name, string $outcome, int $exitCode, int $batchSize) use ($run): void { + $output = $run($exitCode, ['--filter', "($name|passes)", "--stop-on-$flag=2", "--max-batch-size=$batchSize"]); + $plural = in_array($outcome, ['warning', 'notice'], true) ? $outcome.'s' : $outcome; + + expect($output)->toContain("Tests: 2 $plural (") + ->not->toMatch('/\d+ passed/'); +})->with($outcomes)->with([0, 1])->skipOnWindows(); + +test('parallel stop-on-defect includes errors warnings and risky tests', function (string $name, string $outcome, int $exitCode) use ($run): void { + $output = $run($exitCode, ['--filter', "($name|passes)", '--stop-on-defect', '--max-batch-size=1']); + + expect($output)->toContain("Tests: 1 $outcome (") + ->not->toMatch('/\d+ passed/'); +})->with([ + ['errors', 'failed', 2], + ['warns', 'warning', 0], + ['is risky', 'risky', 1], +])->skipOnWindows(); + +test('parallel stop-on-defect allows incomplete and skipped tests', function (string $name, string $outcome) use ($run): void { + $output = $run(0, ['--filter', "($name|passes)", '--stop-on-defect']); + + expect($output)->toContain("Tests: 2 $outcome, 10 passed"); +})->with([ + ['is incomplete', 'incomplete'], + ['is skipped', 'skipped'], +])->skipOnWindows(); + +test('parallel does not count the same failure twice', function (int $batchSize) use ($run): void { + $output = $run(1, ['--filter', '(fails|passes)', '--stop-on-failure=3', "--max-batch-size=$batchSize"]); + + expect($output)->toContain('Tests: 2 failed, 10 passed'); +})->with([0, 1])->skipOnWindows(); + +test('parallel counts issues hidden by the progress output', function (string $flag, string $name, string $outcome, int $exitCode) use ($run): void { + $output = $run($exitCode, ['--filter', "($name|passes)", $flag, '--max-batch-size=1']); + + expect($output)->toContain("Tests: 1 $outcome (") + ->not->toMatch('/\d+ passed/'); +})->with([ + ['--stop-on-warning', 'has combined issues', 'failed', 1], + ['--stop-on-defect=2', 'has combined issues', 'failed', 1], + ['--stop-on-warning=2', 'has repeated warnings', 'warning', 0], + ['--stop-on-notice=2', 'has repeated notifications', 'notice', 0], +])->skipOnWindows(); + +test('parallel stops only on the requested deprecation message', function (string $message, int $count) use ($run): void { + $output = $run(1, ['--filter', '(deprecates|passes)', "--stop-on-deprecation=$message", '--max-batch-size=1']); + + expect($output)->toContain("Tests: $count deprecated"); + + if ($count === 1) { + expect($output)->not->toMatch('/\d+ passed/'); + } else { + expect($output)->toContain('10 passed'); + } +})->with([ + ['user deprecation', 1], + ['another deprecation', 2], +])->skipOnWindows(); + +test('parallel honors stop-on flags from XML', function () use ($run): void { + $output = $run(1, ['--configuration=tests/Fixtures/Suites/ParallelStopOn.xml', '--filter', '(fails|passes)', '--max-batch-size=1']); + + expect($output)->toContain('Tests: 1 failed (') + ->not->toMatch('/\d+ passed/'); +})->skipOnWindows(); + +test('parallel CLI thresholds override XML stop-on flags', function () use ($run): void { + $output = $run(1, ['--configuration=tests/Fixtures/Suites/ParallelStopOn.xml', '--filter', '(fails|passes)', '--stop-on-failure=2']); + + expect($output)->toContain('Tests: 2 failed (') + ->not->toMatch('/\d+ passed/'); +})->skipOnWindows(); + +test('parallel clamps empty and non-positive thresholds like PHPUnit', function (string $threshold) use ($run): void { + $output = $run(1, ['--filter', '(fails|passes)', "--stop-on-failure=$threshold"]); + + expect($output)->toContain('Tests: 1 failed (') + ->not->toMatch('/\d+ passed/'); +})->with(['', '0', '-2'])->skipOnWindows(); + +test('parallel counts a shared issue only once across recycled workers', function (string $flag, string $name, string $outcome) use ($run): void { + $output = $run(0, ['--filter', "($name|passes)", "--stop-on-$flag=2", '--max-batch-size=1']); + + expect($output)->toMatch('/Tests:\s+\d+ '.$outcome.'s?, 10 passed/'); +})->with([ + ['warning', 'shares warning', 'warning'], + ['defect', 'shares warning', 'warning'], + ['notice', 'shares notice', 'notice'], +])->skipOnWindows(); + +test('parallel combines stop-on counts from different workers and cleans up their files', function () use ($run): void { + $directory = sys_get_temp_dir().'/pest-stop-on-'.bin2hex(random_bytes(8)); + mkdir($directory); + + try { + $output = $run( + 1, + ['--processes=2', '--stop-on-failure=2', '--filter', '(fails|passes)', '--tmp-dir='.$directory], + ['PEST_TEST_STOP_BARRIER' => $directory], + ); + + expect($output)->toContain('Tests: 2 failed') + ->not->toContain('10 passed') + ->not->toContain('did not reach the barrier') + ->and(glob($directory.'/worker-*'))->toHaveCount(2) + ->and(glob($directory.'/paratest_*'))->toBeEmpty(); + } finally { + foreach (glob($directory.'/worker-*') as $file) { + unlink($file); + } + + rmdir($directory); + } +})->skipOnWindows();