Honor requested shell timeouts and run evals in parallel

Author: TheGreatAxios

CHANGELOG.md

@@ -11,6 +11,16 @@ matching `## [X.Y.Z]` section (plus install instructions). Do not maintain
 parallel copies under `docs/` or `scripts/notes/`. At cut time: rename
 `## [Unreleased]` to `## [X.Y.Z] - YYYY-MM-DD`, then run the release script.
 
+## [Unreleased]
+
+### Plugins
+
+- **Requested `run_shell` timeouts are no longer capped at 10 minutes.** The 15s
+  default when timeout is omitted is unchanged. A ceiling applies only when
+  settings set `shell.maxTimeoutMs`. Capability evals accept `--concurrency <n>`
+  (env `CORBITS_EVAL_CONCURRENCY`, default 1) so a live matrix can run
+  independent case×variant×repeat cells in parallel.
+
 ## [0.2.99] - 2026-08-21
 
 Skywalker is the primary orchestrator over a closed director fleet: product write tools stay off the primary, and you cannot spawn Skywalker as a task leaf. Workers are not done until they return the four-heading report. First-party action skills ship as slashes; eval runners require an explicit provider/model pair; the style skill no longer refuses non-git folders.

evals/capability/README.md

@@ -117,6 +117,9 @@ bun run eval:capability -- \
   --matrix "xai:grok-4.5,openai:gpt-4.1" \
   --out evals/capability/results/matrix.json
 
+# Faster live matrix (independent cells; default is serial)
+bun run eval:capability -- --provider <name> --model <id> --concurrency 4
+
 # Labeled variants
 bun run eval:capability -- --matrix "fast=xai:grok-4.5,strong=openai:gpt-4.1"
 
@@ -165,6 +168,7 @@ Flags:
 | `--agent-timeout-ms <n>` | Wall-clock limit for `runExec` (default `600000`, env `CORBITS_EVAL_AGENT_TIMEOUT_MS`) |
 | `--verify-timeout-ms <n>` | Wall-clock limit for `verify.sh` (default `120000`, env `CORBITS_EVAL_VERIFY_TIMEOUT_MS`) |
 | `--repeats <n>` | Runs per case×variant cell (default `1`; gate runs use `5`, baseline freezes `3`). Results record every repeat plus per-cell aggregates |
+| `--concurrency <n>` | Independent case×variant×repeat cells in parallel (default `1`, env `CORBITS_EVAL_CONCURRENCY`). Each cell still uses its own temp workdir. Use `--concurrency 4` (or similar) to run a live matrix faster |
 | `--dry-run` | Load cases × variants and print plan; no inference. Still requires `--provider`/`--model` or `--matrix` |
 
 ## Case format

scripts/eval-capability.test.ts

@@ -1,15 +1,33 @@
-import { afterEach, describe, expect, test } from "bun:test";
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
 import { mkdir, mkdtemp, writeFile, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
 
-import { initEvalGitRepo, parseArgs } from "./eval-capability.ts";
+import { initEvalGitRepo, mapPool, parseArgs } from "./eval-capability.ts";
 
 const execFileAsync = promisify(execFile);
 
 describe("parseArgs", () => {
+  const savedConcurrency = process.env.CORBITS_EVAL_CONCURRENCY;
+
+  const restoreConcurrency = (): void => {
+    if (savedConcurrency === undefined) {
+      delete process.env.CORBITS_EVAL_CONCURRENCY;
+    } else {
+      process.env.CORBITS_EVAL_CONCURRENCY = savedConcurrency;
+    }
+  };
+
+  afterEach(() => {
+    restoreConcurrency();
+  });
+
+  beforeEach(() => {
+    delete process.env.CORBITS_EVAL_CONCURRENCY;
+  });
+
   test("--help does not require provider or model", () => {
     const opts = parseArgs(["--help"]);
     expect(opts.help).toBe(true);
@@ -61,6 +79,77 @@ describe("parseArgs", () => {
     expect(pair.provider).toBe("foo");
     expect(pair.model).toBe("bar");
   });
+
+  test("defaults concurrency to 1", () => {
+    delete process.env.CORBITS_EVAL_CONCURRENCY;
+    const opts = parseArgs(["--provider", "foo", "--model", "bar"]);
+    expect(opts.concurrency).toBe(1);
+  });
+
+  test("--concurrency 4 is accepted", () => {
+    delete process.env.CORBITS_EVAL_CONCURRENCY;
+    const opts = parseArgs(["--provider", "foo", "--model", "bar", "--concurrency", "4"]);
+    expect(opts.concurrency).toBe(4);
+  });
+
+  test("invalid --concurrency values throw", () => {
+    const pair = ["--provider", "foo", "--model", "bar"] as const;
+    expect(() => parseArgs([...pair, "--concurrency", "0"])).toThrow(/positive integer/);
+    expect(() => parseArgs([...pair, "--concurrency", "-1"])).toThrow(/positive integer/);
+    expect(() => parseArgs([...pair, "--concurrency", "1.5"])).toThrow(/positive integer/);
+    expect(() => parseArgs([...pair, "--concurrency", "foo"])).toThrow(/positive integer/);
+  });
+
+  test("CORBITS_EVAL_CONCURRENCY sets the default", () => {
+    process.env.CORBITS_EVAL_CONCURRENCY = "3";
+    const opts = parseArgs(["--provider", "foo", "--model", "bar"]);
+    expect(opts.concurrency).toBe(3);
+  });
+
+  test("--concurrency overrides CORBITS_EVAL_CONCURRENCY", () => {
+    process.env.CORBITS_EVAL_CONCURRENCY = "8";
+    const opts = parseArgs(["--provider", "foo", "--model", "bar", "--concurrency", "2"]);
+    expect(opts.concurrency).toBe(2);
+  });
+
+  test("invalid CORBITS_EVAL_CONCURRENCY throws", () => {
+    process.env.CORBITS_EVAL_CONCURRENCY = "0";
+    expect(() => parseArgs(["--provider", "foo", "--model", "bar"])).toThrow(
+      /CORBITS_EVAL_CONCURRENCY must be a positive integer/,
+    );
+  });
+});
+
+describe("mapPool", () => {
+  test("N overlapping jobs with concurrency N finish in ~one job duration", async () => {
+    const jobMs = 80;
+    const n = 4;
+    const start = Date.now();
+    const results = await mapPool([0, 1, 2, 3], n, async (item) => {
+      await new Promise((r) => setTimeout(r, jobMs));
+      return item;
+    });
+    const elapsed = Date.now() - start;
+    expect(results).toEqual([0, 1, 2, 3]);
+    expect(elapsed).toBeLessThan(jobMs * 2);
+    expect(elapsed).toBeGreaterThanOrEqual(jobMs - 20);
+  });
+
+  test("preserves input order when later items finish first", async () => {
+    const results = await mapPool([1, 2, 3], 3, async (item) => {
+      await new Promise((r) => setTimeout(r, (4 - item) * 30));
+      return item;
+    });
+    expect(results).toEqual([1, 2, 3]);
+  });
+
+  test("empty input returns an empty array", async () => {
+    expect(await mapPool([], 4, async (item) => item)).toEqual([]);
+  });
+
+  test("rejects non-positive concurrency", async () => {
+    await expect(mapPool([1], 0, async (item) => item)).rejects.toThrow(/positive integer/);
+  });
 });
 
 describe("initEvalGitRepo", () => {

scripts/eval-capability.ts

@@ -68,6 +68,8 @@ type CliOptions = {
   verifyTimeoutMs: number;
   /** Runs per case×variant cell (gate runs use 5; freeze runs use 3). */
   repeats: number;
+  /** Independent case×variant×repeat cells in parallel (default 1). */
+  concurrency: number;
   dryRun: boolean;
   help: boolean;
   /**
@@ -93,18 +95,62 @@ function printUsage(): void {
   --agent-timeout-ms <n> Wall-clock limit for runExec (default 1200000)
   --verify-timeout-ms <n> Wall-clock limit for verify.sh (default 120000)
   --repeats <n>         Runs per case×variant cell (default 1; gate runs use 5)
+  --concurrency <n>     Independent cells in parallel (default 1, env CORBITS_EVAL_CONCURRENCY)
   --dry-run             List cases × variants only (still requires --provider/--model or --matrix)
   --allow-provider-fallback  Allow resolved provider/model to differ from
                              what was requested (default: hard-fail)
   -h, --help            Show help
 `);
 }
 
+function parsePositiveInteger(raw: string, label: string): number {
+  const n = Number(raw);
+  if (!Number.isInteger(n) || n <= 0) {
+    throw new Error(`${label} must be a positive integer`);
+  }
+  return n;
+}
+
+function defaultConcurrency(): number {
+  const raw = process.env.CORBITS_EVAL_CONCURRENCY;
+  if (raw === undefined || raw === "") return 1;
+  return parsePositiveInteger(raw, "CORBITS_EVAL_CONCURRENCY");
+}
+
+/**
+ * Run `mapper` over `items` with at most `concurrency` in flight.
+ * Results stay in input order even when later items finish first.
+ */
+export async function mapPool<T, R>(
+  items: readonly T[],
+  concurrency: number,
+  mapper: (item: T, index: number) => Promise<R>,
+): Promise<R[]> {
+  if (!Number.isInteger(concurrency) || concurrency <= 0) {
+    throw new Error("concurrency must be a positive integer");
+  }
+  if (items.length === 0) return [];
+  const results: R[] = new Array(items.length);
+  let nextIndex = 0;
+  const worker = async (): Promise<void> => {
+    while (true) {
+      const index = nextIndex;
+      nextIndex += 1;
+      if (index >= items.length) return;
+      results[index] = await mapper(items[index]!, index);
+    }
+  };
+  const workerCount = Math.min(concurrency, items.length);
+  await Promise.all(Array.from({ length: workerCount }, () => worker()));
+  return results;
+}
+
 export function parseArgs(argv: readonly string[]): CliOptions {
   const opts: CliOptions = {
     caseSelector: "all",
     skipPermissions: true,
     repeats: 1,
+    concurrency: defaultConcurrency(),
     dryRun: false,
     help: false,
     allowProviderFallback: false,
@@ -175,6 +221,9 @@ export function parseArgs(argv: readonly string[]): CliOptions {
         opts.repeats = n;
         break;
       }
+      case "--concurrency":
+        opts.concurrency = parsePositiveInteger(next(), "--concurrency");
+        break;
       case "--dry-run":
         opts.dryRun = true;
         break;
@@ -771,6 +820,7 @@ async function main(): Promise<number> {
   }
 
   console.log(`Repeats per cell: ${opts.repeats}`);
+  console.log(`Concurrency: ${opts.concurrency}`);
 
   if (opts.dryRun) {
     console.log("dry-run: no inference");
@@ -781,14 +831,15 @@ async function main(): Promise<number> {
   }
 
   const startedAt = new Date().toISOString();
-  const results: CaseResult[] = [];
-
+  const cells: Array<{ caseDef: EvalCase; variant: EvalVariant; repeat: number }> = [];
   for (const { caseDef, variant } of plan) {
     for (let repeat = 0; repeat < opts.repeats; repeat++) {
-      const result = await runCase(caseDef, variant, opts, repeat);
-      results.push(result);
+      cells.push({ caseDef, variant, repeat });
     }
   }
+  const results = await mapPool(cells, opts.concurrency, ({ caseDef, variant, repeat }) =>
+    runCase(caseDef, variant, opts, repeat),
+  );
 
   const finishedAt = new Date().toISOString();
   const totals = summarizeRun(results);

src/plugins/shell-guard-plugin.test.ts

@@ -13,6 +13,7 @@ import {
   DEFAULT_SHELL_TIMEOUT_MS,
   MAX_SHELL_OUTPUT_BYTES,
   advertiseShellGuardTimeout,
+  resolveShellTimeoutMs,
   runGuardedShell,
   shellGuardPlugin,
 } from "./shell-guard-plugin.js";
@@ -154,6 +155,36 @@ describe("runGuardedShell", () => {
   });
 });
 
+describe("resolveShellTimeoutMs", () => {
+  test("omitted timeout uses the 15s default", () => {
+    expect(resolveShellTimeoutMs(undefined, DEFAULT_SHELL_TIMEOUT_MS)).toBe(15_000);
+    expect(resolveShellTimeoutMs(undefined, DEFAULT_SHELL_TIMEOUT_MS, undefined)).toBe(
+      DEFAULT_SHELL_TIMEOUT_MS,
+    );
+  });
+
+  test("non-positive requested timeout falls back to default", () => {
+    expect(resolveShellTimeoutMs(0, DEFAULT_SHELL_TIMEOUT_MS)).toBe(DEFAULT_SHELL_TIMEOUT_MS);
+    expect(resolveShellTimeoutMs(-1, DEFAULT_SHELL_TIMEOUT_MS)).toBe(DEFAULT_SHELL_TIMEOUT_MS);
+  });
+
+  test("requested timeout well above 10 minutes is not clamped when maxMs is omitted", () => {
+    expect(resolveShellTimeoutMs(5_400_000, DEFAULT_SHELL_TIMEOUT_MS)).toBe(5_400_000);
+    expect(resolveShellTimeoutMs(5_400_000, DEFAULT_SHELL_TIMEOUT_MS, undefined)).toBe(5_400_000);
+    expect(resolveShellTimeoutMs(900_000, DEFAULT_SHELL_TIMEOUT_MS)).toBe(900_000);
+  });
+
+  test("configured maxMs still clamps", () => {
+    expect(resolveShellTimeoutMs(900_000, DEFAULT_SHELL_TIMEOUT_MS, 100)).toBe(100);
+    expect(resolveShellTimeoutMs(5_400_000, DEFAULT_SHELL_TIMEOUT_MS, 600_000)).toBe(600_000);
+    expect(resolveShellTimeoutMs(undefined, DEFAULT_SHELL_TIMEOUT_MS, 100)).toBe(100);
+  });
+
+  test("requested below maxMs is unchanged", () => {
+    expect(resolveShellTimeoutMs(1_000, DEFAULT_SHELL_TIMEOUT_MS, 600_000)).toBe(1_000);
+  });
+});
+
 describe("advertiseShellGuardTimeout", () => {
   test("rewrites run_shell timeout default to match the guard", () => {
     const rewritten = advertiseShellGuardTimeout({

src/plugins/shell-guard-plugin.ts

@@ -18,9 +18,6 @@ import {
 // so open-ended walks cannot OOM the host.
 
 export const DEFAULT_SHELL_TIMEOUT_MS = 15_000;
-// Upper bound on a per-command timeout override, so the model cannot ask for an
-// effectively unbounded wait. Configurable via settings.
-export const MAX_SHELL_TIMEOUT_MS = 600_000;
 export const MAX_SHELL_OUTPUT_BYTES = 512_000;
 
 export type ShellTimeoutConfig = {
@@ -29,6 +26,21 @@ export type ShellTimeoutConfig = {
   maxOutputBytes?: number;
 };
 
+/**
+ * Effective run_shell timeout. Omitting `requested` (or a non-positive value)
+ * uses `defaultMs`. `maxMs` clamps only when settings pass it — there is no
+ * implicit 10-minute ceiling.
+ */
+export function resolveShellTimeoutMs(
+  requested: number | undefined,
+  defaultMs: number,
+  maxMs?: number,
+): number {
+  const base = requested !== undefined && requested > 0 ? requested : defaultMs;
+  if (maxMs === undefined) return base;
+  return Math.min(base, maxMs);
+}
+
 /**
  * Stock tools-posix still advertises timeout default 30000. Shell-guard enforces
  * 15s default; rewrite the definition the model sees so schema and behavior agree.
@@ -356,7 +368,6 @@ export function shellGuardPlugin(
   options: ShellGuardPluginOptions = {},
 ): ToolPlugin {
   const defaultMs = timeoutConfig?.defaultMs ?? DEFAULT_SHELL_TIMEOUT_MS;
-  const maxMs = timeoutConfig?.maxMs ?? MAX_SHELL_TIMEOUT_MS;
   const maxOutputBytes = timeoutConfig?.maxOutputBytes ?? MAX_SHELL_OUTPUT_BYTES;
   const sessionRoot = realpathSync(cwd);
   let retainedShellCwd = sessionRoot;
@@ -412,9 +423,11 @@ export function shellGuardPlugin(
             };
           }
           const requested = optionalNumber(call.arguments.timeout);
-          const baseTimeoutMs =
-            requested !== undefined && requested > 0 ? requested : defaultMs;
-          const effectiveTimeout = Math.min(baseTimeoutMs, maxMs);
+          const effectiveTimeout = resolveShellTimeoutMs(
+            requested,
+            defaultMs,
+            timeoutConfig?.maxMs,
+          );
           const wrappedCommand = wrapCommandWithPwdProbe(command);
           try {
             const { output, exitCode, timedOut, outputTruncated } =