test: stabilize and speed the parallel suite without a full rerun

.github/workflows/ci.yml

@@ -75,7 +75,7 @@ jobs:
         run: cd gui && bun run build
 
       - name: Test suite
-        run: bun test --isolate tests
+        run: bun run test:parallel
 
       - name: CLI help smoke
         run: bun run src/cli/index.ts help

AGENTS.md

@@ -63,6 +63,11 @@ Run `bun run typecheck` and `bun run test:parallel` (fall back to
 `bun run test` if the parallel runner misbehaves) before proposing or
 approving any non-trivial change. CI runs these on Linux, Windows, and macOS.
 
+If `test:parallel` reports failed files, rerun **only those files**
+(`bun run test:parallel tests/foo.test.ts …`). The runner already retries
+failed files once on a single worker. Do not recover from a load flake by
+rerunning the entire suite at `CCX_TEST_PARALLEL_WORKERS=2`.
+
 ## Issues and pull requests (agents)
 
 Agent-created issues must use the repository templates. Pull requests use

README.md

@@ -313,7 +313,8 @@ Source development requires the `bun` CLI on your `PATH`.
 cd /path/to/CodexCommander
 bun install
 bun run typecheck
-bun run test:parallel   # preferred — much faster (parallel runner)
+bun run test:parallel   # preferred — batches cheap files, isolates server/spawn files,
+                        # retries only failures (do not rerun the whole suite on a flake)
 bun run test            # serial fallback
 ```
 

docs-site/src/content/docs/contributing.md

@@ -36,8 +36,9 @@ bun run prepare:package           # refresh package launchers/assets
 
 Most tests are flat `tests/*.test.ts` Bun tests. `tests/helpers/` contains shared fixtures and
 `tests/e2e-style/` contains broader native-parity scenarios. Keep a focused regression near the
-existing tests for the subsystem you change; run the full suite for shared routing, adapters, config,
-or server behavior.
+existing tests for the subsystem you change; run `bun run test:parallel` for shared routing,
+adapters, config, or server behavior. If that reports failed files, rerun only those files — not
+the entire suite. The parallel runner retries failed files once automatically.
 
 The docs site you're reading lives in `docs-site/` (Astro + Starlight):
 
@@ -78,7 +79,8 @@ contributor work.
   and results). Empty or placeholder-only descriptions are not enough for review.
 - If the change touches the dashboard UI, include a screenshot in the description.
 - Behavior changes need a focused regression near the existing tests for that subsystem.
-  Shared routing, adapter, config, or server changes need the full suite green.
+  Shared routing, adapter, config, or server changes need `bun run test:parallel` green.
+  On a flake, rerun only the failed files; do not rerun the entire suite.
 
 The retired dual-track Go native port is not part of this repository. Bun-native TypeScript on
 `main` is the single runtime line.

scripts/test-parallel.ts

@@ -1,28 +1,33 @@
 /**
  * Parallel full-suite runner.
  *
- * `scripts/test.ts` runs one fresh process per test file (shard size 1) because
- * Bun 1.3.x on macOS can leave the event loop spinning after a server-heavy file
- * completes, stalling the next file in the same process. That isolation is
- * preserved here: each file still gets its own `bun test --isolate` process,
- * but up to `CCX_TEST_PARALLEL_WORKERS` of them run at the same time.
+ * Cheap unit files share a process (batched, `--no-isolate`, one test at a
+ * time). Server/subprocess files still get a dedicated `bun test --isolate`
+ * process — Bun 1.3.x on macOS can leave the event loop spinning after those,
+ * which is why the serial runner uses shard size 1.
  *
- * On a multi-core machine this turns the serial ~40-minute full suite into a
- * few minutes. Default worker count is min(4, CPU count); override with
- * `CCX_TEST_PARALLEL_WORKERS`. Optional positional args restrict the run to the
- * given test files (useful for smoke probes).
+ * Default worker count is min(4, CPU count); override with
+ * `CCX_TEST_PARALLEL_WORKERS`. Batch size defaults to 8 (`CCX_TEST_BATCH_SIZE`).
+ * `CCX_TEST_FORCE_ISOLATE=1` restores one process per file.
  *
- * The same exclusivity queue as the serial runner applies: this script waits
- * for any other `bun test --isolate` runner to finish before starting, so two
- * suites never fight over the CPU.
+ * Failed items are retried once, file-by-file, on a single worker. That is the
+ * load-flake recovery path: do not rerun the entire suite. Optional positional
+ * args restrict the run to the given test files (including retry-only reruns).
+ *
+ * The same exclusivity queue as the serial runner applies.
  */
 import { availableParallelism } from "node:os";
+import { relative } from "node:path";
 import {
   createIsolatedTestEnvironment,
-  findCompetingTestRunners,
   listRepositoryTestFiles,
   waitForExclusiveRun,
 } from "./test";
+import {
+  filesFromFailedPlanItems,
+  planParallelTests,
+  resolveTestBatchSize,
+} from "./test-plan";
 
 export function resolveWorkerCount(
   raw = process.env.CCX_TEST_PARALLEL_WORKERS,
@@ -38,11 +43,38 @@ export function resolveWorkerCount(
   return parsed;
 }
 
-function runIsolatedFile(file: string): Promise<number> {
+export function resolveRetryCount(raw = process.env.CCX_TEST_RETRY): number {
+  if (raw === undefined || raw.trim() === "") return 1;
+  const parsed = Number(raw);
+  if (!Number.isSafeInteger(parsed) || parsed < 0) {
+    throw new Error(`CCX_TEST_RETRY must be a non-negative integer, received ${JSON.stringify(raw)}`);
+  }
+  return parsed;
+}
+
+interface WorkItem {
+  files: string[];
+  isolate: boolean;
+}
+
+function displayPath(file: string): string {
+  const rel = relative(process.cwd(), file);
+  return rel && !rel.startsWith("..") ? rel : file;
+}
+
+function displayItem(item: WorkItem): string {
+  if (item.files.length === 1) return displayPath(item.files[0]!);
+  return `${item.files.length} files (${displayPath(item.files[0]!)} …)`;
+}
+
+function runWorkItem(item: WorkItem): Promise<number> {
   const isolated = createIsolatedTestEnvironment();
   const memoryArgs = process.env.CCX_TEST_SMOL === "1" ? ["--smol"] : [];
+  const isolateArgs = item.isolate
+    ? ["--isolate"]
+    : ["--no-isolate", "--max-concurrency=1"];
   return Bun.spawn(
-    [process.execPath, "test", "--isolate", ...memoryArgs, file],
+    [process.execPath, "test", ...isolateArgs, ...memoryArgs, ...item.files],
     {
       env: isolated.env,
       stdin: "inherit",
@@ -55,44 +87,88 @@ function runIsolatedFile(file: string): Promise<number> {
   });
 }
 
+async function runQueue(items: WorkItem[], workers: number, label: string): Promise<WorkItem[]> {
+  const workerCount = Math.min(workers, Math.max(1, items.length));
+  let next = 0;
+  const failures: WorkItem[] = [];
+
+  const workerLoop = async (): Promise<void> => {
+    for (;;) {
+      const index = next++;
+      if (index >= items.length) return;
+      const item = items[index]!;
+      const code = await runWorkItem(item);
+      if (code !== 0) failures.push(item);
+      console.warn(
+        `[test:parallel] ${label} ${index + 1}/${items.length} ${code === 0 ? "ok" : "FAIL"} ${displayItem(item)}`,
+      );
+    }
+  };
+
+  await Promise.all(Array.from({ length: workerCount }, () => workerLoop()));
+  return failures;
+}
+
 if (import.meta.main) {
   const requested = process.argv.slice(2);
   await waitForExclusiveRun(process.pid);
 
   const files = requested.length > 0 ? requested : listRepositoryTestFiles();
   if (files.length === 0) throw new Error("no test files found");
-  const workers = Math.min(resolveWorkerCount(), files.length);
+
+  const plan = planParallelTests(files);
+  const items: WorkItem[] = [
+    ...plan.batches.map(batch => ({ files: batch, isolate: false })),
+    ...plan.isolate.map(file => ({ files: [file], isolate: true })),
+  ];
+  const workers = Math.min(resolveWorkerCount(), items.length);
+  const retryCount = resolveRetryCount();
 
   console.warn(
-    `[test:parallel] ${files.length} file(s) across ${workers} worker(s)`
-    + (requested.length > 0 ? " (explicit file list)" : " (CCX_TEST_PARALLEL_WORKERS to override)"),
+    `[test:parallel] ${files.length} file(s): ${plan.isolate.length} isolated, ${plan.batches.length} batch(es) `
+    + `of up to ${resolveTestBatchSize()} across ${workers} worker(s)`
+    + (requested.length > 0 ? " (explicit file list)" : " (CCX_TEST_PARALLEL_WORKERS / CCX_TEST_BATCH_SIZE to override)"),
   );
 
   const startedAt = Date.now();
-  let next = 0;
-  let failed = 0;
-  const failures: string[] = [];
+  let failures = await runQueue(items, workers, "run");
+  const recovered: string[] = [];
 
-  const workerLoop = async (): Promise<void> => {
-    for (;;) {
-      const index = next++;
-      if (index >= files.length) return;
-      const file = files[index]!;
-      const code = await runIsolatedFile(file);
-      if (code !== 0) {
-        failed += 1;
-        failures.push(file);
+  if (failures.length > 0 && retryCount > 0) {
+    const retryFiles = filesFromFailedPlanItems(failures);
+    console.warn(
+      `[test:parallel] ${retryFiles.length} file(s) failed; retrying only those files (${retryCount} pass(es), 1 worker):`,
+    );
+    for (const file of retryFiles) console.warn(`  ${displayPath(file)}`);
+
+    const stillFailing = new Set(retryFiles);
+    for (let attempt = 1; attempt <= retryCount; attempt += 1) {
+      const retryItems: WorkItem[] = [...stillFailing].map(file => ({ files: [file], isolate: true }));
+      const retryFailures = await runQueue(retryItems, 1, `retry ${attempt}/${retryCount}`);
+      const failedThisPass = new Set(filesFromFailedPlanItems(retryFailures));
+      for (const file of stillFailing) {
+        if (!failedThisPass.has(file)) recovered.push(file);
       }
-      console.warn(`[test:parallel] ${index + 1}/${files.length} ${code === 0 ? "ok" : "FAIL"} ${file}`);
+      stillFailing.clear();
+      for (const file of failedThisPass) stillFailing.add(file);
+      if (stillFailing.size === 0) break;
     }
-  };
+    failures = [...stillFailing].map(file => ({ files: [file], isolate: true }));
+  }
 
-  await Promise.all(Array.from({ length: workers }, () => workerLoop()));
   const minutes = ((Date.now() - startedAt) / 60_000).toFixed(1);
 
+  if (recovered.length > 0) {
+    console.warn(`[test:parallel] recovered after retry (${recovered.length} file(s)):`);
+    for (const file of recovered) console.warn(`  ${displayPath(file)}`);
+  }
+
   if (failures.length > 0) {
-    console.error(`[test:parallel] ${failures.length} file(s) failed after ${minutes} min:`);
-    for (const file of failures) console.error(`  ${file}`);
+    const failedFiles = filesFromFailedPlanItems(failures);
+    console.error(`[test:parallel] ${failedFiles.length} file(s) failed after ${minutes} min:`);
+    for (const file of failedFiles) console.error(`  ${displayPath(file)}`);
+    console.error("[test:parallel] rerun only the failed files, not the entire suite:");
+    console.error(`  bun run test:parallel ${failedFiles.map(displayPath).join(" ")}`);
     process.exit(1);
   }
   console.warn(`[test:parallel] all ${files.length} file(s) passed in ${minutes} min`);

scripts/test-plan.ts

@@ -0,0 +1,93 @@
+import { readFileSync } from "node:fs";
+
+/**
+ * Cheap files share a process; server/subprocess files stay isolated.
+ *
+ * `bun test --isolate` is a fresh global object, not a fresh OS process. The
+ * serial and parallel wrappers spawn one process per file because Bun 1.3.x on
+ * macOS can leave the event loop spinning after a server-heavy file, stalling
+ * whatever would run next in that process. That isolation is necessary — it is
+ * also why 600 one-file processes thrash a laptop.
+ *
+ * Files that never bind a server, spawn `process.execPath`, or write the
+ * process HOME sandbox do not hit that stall. Batching them with
+ * `--no-isolate --max-concurrency=1` keeps HOME sandboxing (preload still
+ * runs once per process) without paying a Bun boot per file. Isolate-needed
+ * files still get their own process.
+ */
+export const DEFAULT_TEST_BATCH_SIZE = 8;
+
+export type TestLane = "batch" | "isolate";
+
+export interface ParallelTestPlan {
+  isolate: string[];
+  batches: string[][];
+}
+
+/**
+ * Isolate anything that cannot safely share a process with the next file:
+ * - server binds / nested Bun CLI spawns (macOS event-loop stall + load timeouts)
+ * - writes to the process HOME sandbox (preload HOME is per-process, not per-file)
+ */
+const ISOLATE_SOURCE_RE =
+  /\bstartServer\s*\(|\bBun\.serve\s*\(|\bprocess\.execPath\b|\bsaveConfig\s*\(|\binstallIsolatedCodexHome\s*\(|process\.env\.(CODEXCOMMANDER_HOME|CODEX_HOME)\s*=/;
+
+export function classifyTestFile(filePath: string, source: string): TestLane {
+  const normalized = filePath.replaceAll("\\", "/");
+  if (normalized.includes("/e2e-style/")) return "isolate";
+  if (ISOLATE_SOURCE_RE.test(source)) return "isolate";
+  return "batch";
+}
+
+export function resolveTestBatchSize(raw = process.env.CCX_TEST_BATCH_SIZE): number {
+  if (raw === undefined || raw.trim() === "") return DEFAULT_TEST_BATCH_SIZE;
+  const parsed = Number(raw);
+  if (!Number.isSafeInteger(parsed) || parsed < 1) {
+    throw new Error(`CCX_TEST_BATCH_SIZE must be a positive integer, received ${JSON.stringify(raw)}`);
+  }
+  return parsed;
+}
+
+export function forceIsolateAllFiles(raw = process.env.CCX_TEST_FORCE_ISOLATE): boolean {
+  return raw === "1";
+}
+
+export function planParallelTests(
+  files: readonly string[],
+  options: { batchSize?: number; forceIsolate?: boolean; readSource?: (file: string) => string } = {},
+): ParallelTestPlan {
+  const batchSize = options.batchSize ?? resolveTestBatchSize();
+  const forceIsolate = options.forceIsolate ?? forceIsolateAllFiles();
+  const readSource = options.readSource ?? ((file: string) => readFileSync(file, "utf8"));
+
+  const isolate: string[] = [];
+  const batchable: string[] = [];
+  for (const file of files) {
+    if (forceIsolate || classifyTestFile(file, readSource(file)) === "isolate") {
+      isolate.push(file);
+    } else {
+      batchable.push(file);
+    }
+  }
+
+  const batches: string[][] = [];
+  for (let offset = 0; offset < batchable.length; offset += batchSize) {
+    batches.push(batchable.slice(offset, offset + batchSize));
+  }
+  return { isolate, batches };
+}
+
+export function filesFromFailedPlanItems(
+  failed: readonly { files: readonly string[] }[],
+): string[] {
+  const seen = new Set<string>();
+  const files: string[] = [];
+  for (const item of failed) {
+    for (const file of item.files) {
+      if (seen.has(file)) continue;
+      seen.add(file);
+      files.push(file);
+    }
+  }
+  return files;
+}

src/AGENTS.md

@@ -23,6 +23,7 @@ This file applies to `src/` and inherits the repository-wide rules in `/AGENTS.m
 
 - Place focused regression coverage near the existing tests for the affected subsystem.
 - For focused behavior, run the relevant `bun test tests/<name>.test.ts` and `bun run typecheck`.
-- For shared routing, adapters, config, OAuth, or server behavior, also run `bun run test`.
+- For shared routing, adapters, config, OAuth, or server behavior, also run `bun run test:parallel`.
+  If that reports failures, rerun only the failed files — not the entire suite.
 - For logging, requests, credentials, account data, or fixtures, also run `bun run privacy:scan`.
 - Update `docs-site/` when the change affects user-visible behavior or configuration.

structure/06_docs-and-release.md

@@ -111,7 +111,7 @@ cd gui && bun run lint
 cd gui && bun run lint:i18n
 cd gui && bun test tests
 cd gui && bun run build
-bun test --isolate tests
+bun run test:parallel
 bun run src/cli/index.ts help
 ```
 

tests/ci-workflows.test.ts

@@ -122,15 +122,15 @@ describe("ci.yml is the only pull-request check", () => {
     for (const step of [
       "bun x tsc --noEmit",
       "bun run privacy:scan",
-      "bun test --isolate tests",
+      "bun run test:parallel",
       "bun run lint",
       "bun run lint:i18n",
       "bun run build",
     ]) {
       expect(ci).toContain(step);
     }
     // The suite serves gui/dist, so the build must precede the test run.
-    expect(ci.indexOf("bun run build")).toBeLessThan(ci.indexOf("bun test --isolate tests"));
+    expect(ci.indexOf("bun run build")).toBeLessThan(ci.indexOf("bun run test:parallel"));
   });
 
   test("grants read-only permissions and keeps no checkout credentials", () => {

tests/claude-models-discovery.test.ts

@@ -6,10 +6,11 @@ import { saveConfig } from "../src/config";
 import { startServer } from "../src/server";
 import type { CodexCommanderConfig } from "../src/types";
 import { installIsolatedCodexHome, type IsolatedCodexHome } from "./helpers/isolated-codex-home";
+import { SERVER_BUDGET_MS } from "./helpers/test-budget";
 
-// Full-suite Windows load: startServer + discovery GETs exceed the default 5s budget
+// startServer + discovery GETs exceed Bun's 5s default under parallel load
 // (same flake class as 810fa115 / claude-management-api).
-setDefaultTimeout(30_000);
+setDefaultTimeout(SERVER_BUDGET_MS);
 
 let testDir = "";
 let previousHome: string | undefined;