perf: Cluster large files into one worker and write node cache incrementally

src/Analyser/AnalysisNodeExtractor.php

@@ -12,6 +12,7 @@
 use PhpParser\NodeVisitor\NameResolver;
 
 use function array_push;
+use function array_slice;
 use function count;
 
 /**
@@ -55,6 +56,7 @@ public function extract(
         $analysisNodeCollector = new AnalysisNodeCollector($this->layerResolver, $withFileAnalysis);
         $nodeTraverser         = new NodeTraverser(new NameResolver(), $analysisNodeCollector);
         $fileAnalyses          = [];
+        $storedCounts          = [0, 0, 0, 0];
 
         foreach ($filesToParse as $fileToParse) {
             try {
@@ -95,6 +97,18 @@ public function extract(
 
                 $progressHandler?->advance($fileToParse);
             }
+
+            // Stored as soon as the file is done: a worker that writes its
+            // whole bucket at the end collides with the others on the file
+            // system, since file creation hardly runs in parallel.
+            if ($this->analysisResultCache instanceof AnalysisResultCache) {
+                $storedCounts = $this->storeFileNodes(
+                    $analysisNodeCollector,
+                    $fileToParse,
+                    $fileAnalyses[$fileToParse] ?? null,
+                    $storedCounts,
+                );
+            }
         }
 
         $extractionResult = new ExtractionResult(
@@ -107,13 +121,48 @@ public function extract(
             $analysisNodeCollector->getAnonymousFunctionNodes(),
         );
 
-        $this->analysisResultCache?->storeExtractionResult(
-            $filesToParse,
+        return $cachedResult->merge($extractionResult);
+    }
+
+    /**
+     * Stores the nodes the collector appended for $file: every node of a file
+     * is appended while that file is traversed, so they are the tail of each
+     * list past the counts stored so far.
+     *
+     * @param array{int, int, int, int} $storedCounts
+     * @return array{int, int, int, int}
+     */
+    private function storeFileNodes(
+        AnalysisNodeCollector $analysisNodeCollector,
+        string $file,
+        ?FileAnalysis $fileAnalysis,
+        array $storedCounts,
+    ): array {
+        [$classNodeCount, $anonymousClassNodeCount, $functionNodeCount, $anonymousFunctionNodeCount] = $storedCounts;
+
+        $classNodes             = $analysisNodeCollector->getClassNodes();
+        $anonymousClassNodes    = $analysisNodeCollector->getAnonymousClassNodes();
+        $functionNodes          = $analysisNodeCollector->getFunctionNodes();
+        $anonymousFunctionNodes = $analysisNodeCollector->getAnonymousFunctionNodes();
+
+        $this->analysisResultCache?->storeAnalysisNodes(
+            $file,
             $this->analysisNodeCacheNamespace,
-            $extractionResult
+            array_slice($classNodes, $classNodeCount),
+            $fileAnalysis,
+            array_slice($anonymousClassNodes, $anonymousClassNodeCount),
+            $analysisNodeCollector->getFileReferences()[$file] ?? [],
+            $analysisNodeCollector->getFileInstantiations()[$file] ?? [],
+            array_slice($functionNodes, $functionNodeCount),
+            array_slice($anonymousFunctionNodes, $anonymousFunctionNodeCount),
         );
 
-        return $cachedResult->merge($extractionResult);
+        return [
+            count($classNodes),
+            count($anonymousClassNodes),
+            count($functionNodes),
+            count($anonymousFunctionNodes),
+        ];
     }
 
     /**

src/Analyser/Parallel/ParallelAnalysisNodeExtractor.php

@@ -20,6 +20,7 @@
 use function array_pop;
 use function array_push;
 use function array_search;
+use function array_sum;
 use function arsort;
 use function count;
 use function dirname;
@@ -326,6 +327,9 @@ public function extract(
      * This minimises the gap between the slowest and fastest worker (makespan), giving near-optimal balance
      * even when one file is much larger than the rest.
      *
+     * The first bucket takes the largest files up to an even share of the bytes before LPT starts, so
+     * one worker pays the memory peak of parsing them instead of every worker getting one of them.
+     *
      * @param list<string> $files
      * @param positive-int $workerCount
      * @return list<list<string>>
@@ -342,8 +346,15 @@ private function buildWorkerBuckets(array $files, int $workerCount): array
         $buckets     = array_fill(0, $workerCount, []);
         $bucketSizes = array_fill(0, $workerCount, 0);
 
+        // The largest files fill one worker up to an even byte share first, so
+        // only that worker pays their AST peaks; the rest are balanced by LPT.
+        $share = array_sum($fileSizes) / $workerCount;
+
         foreach ($fileSizes as $file => $fileSize) {
-            $minIdx                = (int) array_search(min($bucketSizes), $bucketSizes, true);
+            $minIdx = $workerCount > 1 && $bucketSizes[0] < $share
+                ? 0
+                : (int) array_search(min($bucketSizes), $bucketSizes, true);
+
             $buckets[$minIdx][]    = $file;
             $bucketSizes[$minIdx] += $fileSize;
         }