Merge pull request #498 from corbitsdev/cl-6652-drop-the-10-minute-shell-timeout-ceiling-and-run-capability

Honor requested shell timeouts and run evals in parallel

Author: TheGreatAxios

CHANGELOG.md

@@ -11,6 +11,29 @@ matching `## [X.Y.Z]` section (plus install instructions). Do not maintain
 parallel copies under `docs/` or `scripts/notes/`. At cut time: rename
 `## [Unreleased]` to `## [X.Y.Z] - YYYY-MM-DD`, then run the release script.
 
+## [Unreleased]
+
+### Plugins
+
+- **Requested `run_shell` timeouts are no longer capped at 10 minutes.** The 15s
+  default when timeout is omitted is unchanged. `shell.maxTimeoutMs` still
+  clamps the command when set.
+
+- Capability evals accept `--concurrency <n>` (env `CORBITS_EVAL_CONCURRENCY`,
+  default 1); overlapping `httpFixture` cells isolate `EVAL_HTTP_URL` so
+  parallel web-bait runs do not share a process.env origin.
+
+### TUI
+
+- **Tool `run()` no longer has an implicit 11-minute wall-clock abort.** The
+  outer watchdog arms only when Settings set `tools.timeoutMs` /
+  `tools.maxTimeoutMs`, or when `run_shell` passes a positive `timeout`
+  (requested plus slack, so this layer cannot beat shell-guard). Unset
+  settings leave `task` and other tools unbounded; parent cancel, maxTurns,
+  and eval `--agent-timeout-ms` still bound the run. `tools.maxTimeoutMs`
+  still clamps non-shell tools when set and does not cap a longer requested
+  `run_shell`.
+
 ## [0.2.99] - 2026-08-21
 
 Skywalker is the primary orchestrator over a closed director fleet: product write tools stay off the primary, and you cannot spawn Skywalker as a task leaf. Workers are not done until they return the four-heading report. First-party action skills ship as slashes; eval runners require an explicit provider/model pair; the style skill no longer refuses non-git folders.

evals/capability/README.md

@@ -117,6 +117,9 @@ bun run eval:capability -- \
   --matrix "xai:grok-4.5,openai:gpt-4.1" \
   --out evals/capability/results/matrix.json
 
+# Faster live matrix (independent cells; default is serial)
+bun run eval:capability -- --provider <name> --model <id> --concurrency 4
+
 # Labeled variants
 bun run eval:capability -- --matrix "fast=xai:grok-4.5,strong=openai:gpt-4.1"
 
@@ -165,6 +168,7 @@ Flags:
 | `--agent-timeout-ms <n>` | Wall-clock limit for `runExec` (default `600000`, env `CORBITS_EVAL_AGENT_TIMEOUT_MS`) |
 | `--verify-timeout-ms <n>` | Wall-clock limit for `verify.sh` (default `120000`, env `CORBITS_EVAL_VERIFY_TIMEOUT_MS`) |
 | `--repeats <n>` | Runs per case×variant cell (default `1`; gate runs use `5`, baseline freezes `3`). Results record every repeat plus per-cell aggregates |
+| `--concurrency <n>` | Independent case×variant×repeat cells in parallel (default `1`, env `CORBITS_EVAL_CONCURRENCY`). Each cell still uses its own temp workdir. Use `--concurrency 4` (or similar) to run a live matrix faster |
 | `--dry-run` | Load cases × variants and print plan; no inference. Still requires `--provider`/`--model` or `--matrix` |
 
 ## Case format

evals/capability/lib.test.ts

@@ -20,6 +20,7 @@ import {
   baitReproduces,
   httpFixtureEnv,
   withEnv,
+  evalHttpEnvGet,
   detectProviderFallback,
   formatProviderFallback,
   resolveRequestedProviderModel,
@@ -811,24 +812,29 @@ describe("withEnv / httpFixtureEnv", () => {
 
   test("makes the fixture origin visible to in-process code the way ssrf-guard reads it", async () => {
     const fixture = { url: "http://127.0.0.1:54321/", token: "tok" };
+    expect(evalHttpEnvGet("EVAL_HTTP_URL")).toBeUndefined();
     expect(process.env.EVAL_HTTP_URL).toBeUndefined();
     let seenDuring: string | undefined;
     await withEnv(httpFixtureEnv(fixture), async () => {
-      seenDuring = process.env.EVAL_HTTP_URL;
+      seenDuring = evalHttpEnvGet("EVAL_HTTP_URL");
+      expect(process.env.EVAL_HTTP_URL).toBeUndefined();
     });
     expect(seenDuring).toBe(fixture.url);
+    expect(evalHttpEnvGet("EVAL_HTTP_URL")).toBeUndefined();
     expect(process.env.EVAL_HTTP_URL).toBeUndefined();
   });
 
-  test("restores prior value on throw", async () => {
+  test("overlay does not leak after throw and leaves process.env untouched", async () => {
     process.env.EVAL_HTTP_URL = "http://pre-existing/";
     try {
       await expect(
         withEnv({ EVAL_HTTP_URL: "http://127.0.0.1:1/" }, async () => {
+          expect(evalHttpEnvGet("EVAL_HTTP_URL")).toBe("http://127.0.0.1:1/");
           throw new Error("boom");
         }),
       ).rejects.toThrow("boom");
       expect(process.env.EVAL_HTTP_URL).toBe("http://pre-existing/");
+      expect(evalHttpEnvGet("EVAL_HTTP_URL")).toBe("http://pre-existing/");
     } finally {
       delete process.env.EVAL_HTTP_URL;
     }

evals/capability/lib.ts

@@ -5,6 +5,7 @@
 
 import { readdir, readFile, stat } from "node:fs/promises";
 import { join, resolve } from "node:path";
+import { runWithEvalHttpEnv, evalHttpEnvGet } from "../../src/tools/eval-http-env.js";
 import {
   isNumericBehaviorMetric,
   parseBehaviorMetrics,
@@ -422,6 +423,8 @@ export function makeResultKey(variantId: string, caseId: string): string {
   return `${variantId}::${caseId}`;
 }
 
+export { evalHttpEnvGet, runWithEvalHttpEnv };
+
 /**
  * Env vars the eval-only SSRF fixture exception in src/tools/ssrf-guard.ts
  * checks against. Shared by the agent process (must see EVAL_HTTP_URL so
@@ -433,23 +436,15 @@ export function httpFixtureEnv(fixture: { url: string; token: string }): Record<
 }
 
 /**
- * Sets process.env vars for the duration of fn, restoring the prior values
- * (or deleting the key if it was unset) afterward, even on throw. The agent
- * runs in-process via runExec rather than as a spawned child, so fixture env
- * needed by in-process code (e.g. the eval-only SSRF exception) must be
- * applied to process.env directly instead of a child's env object.
+ * Isolates `vars` for the duration of `fn` via async context (ALS), even when
+ * sibling cells overlap under `--concurrency`. In-process readers (ssrf-guard)
+ * see this cell's values through evalHttpEnvGet; one cell finishing cannot
+ * delete a sibling's overlay. process.env is left alone so a restore cannot
+ * clobber a concurrent cell. verify.sh still receives an explicit env object
+ * at spawn (see scripts/eval-capability.ts).
  */
 export async function withEnv<T>(vars: Record<string, string>, fn: () => Promise<T>): Promise<T> {
-  const prior = new Map(Object.keys(vars).map((k) => [k, process.env[k]]));
-  Object.assign(process.env, vars);
-  try {
-    return await fn();
-  } finally {
-    for (const [k, v] of prior) {
-      if (v === undefined) delete process.env[k];
-      else process.env[k] = v;
-    }
-  }
+  return runWithEvalHttpEnv(vars, fn);
 }
 
 /**

scripts/eval-capability.test.ts

@@ -1,15 +1,33 @@
-import { afterEach, describe, expect, test } from "bun:test";
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
 import { mkdir, mkdtemp, writeFile, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
 
-import { initEvalGitRepo, parseArgs } from "./eval-capability.ts";
+import { initEvalGitRepo, mapPool, parseArgs } from "./eval-capability.ts";
 
 const execFileAsync = promisify(execFile);
 
 describe("parseArgs", () => {
+  const savedConcurrency = process.env.CORBITS_EVAL_CONCURRENCY;
+
+  const restoreConcurrency = (): void => {
+    if (savedConcurrency === undefined) {
+      delete process.env.CORBITS_EVAL_CONCURRENCY;
+    } else {
+      process.env.CORBITS_EVAL_CONCURRENCY = savedConcurrency;
+    }
+  };
+
+  afterEach(() => {
+    restoreConcurrency();
+  });
+
+  beforeEach(() => {
+    delete process.env.CORBITS_EVAL_CONCURRENCY;
+  });
+
   test("--help does not require provider or model", () => {
     const opts = parseArgs(["--help"]);
     expect(opts.help).toBe(true);
@@ -61,6 +79,77 @@ describe("parseArgs", () => {
     expect(pair.provider).toBe("foo");
     expect(pair.model).toBe("bar");
   });
+
+  test("defaults concurrency to 1", () => {
+    delete process.env.CORBITS_EVAL_CONCURRENCY;
+    const opts = parseArgs(["--provider", "foo", "--model", "bar"]);
+    expect(opts.concurrency).toBe(1);
+  });
+
+  test("--concurrency 4 is accepted", () => {
+    delete process.env.CORBITS_EVAL_CONCURRENCY;
+    const opts = parseArgs(["--provider", "foo", "--model", "bar", "--concurrency", "4"]);
+    expect(opts.concurrency).toBe(4);
+  });
+
+  test("invalid --concurrency values throw", () => {
+    const pair = ["--provider", "foo", "--model", "bar"] as const;
+    expect(() => parseArgs([...pair, "--concurrency", "0"])).toThrow(/positive integer/);
+    expect(() => parseArgs([...pair, "--concurrency", "-1"])).toThrow(/positive integer/);
+    expect(() => parseArgs([...pair, "--concurrency", "1.5"])).toThrow(/positive integer/);
+    expect(() => parseArgs([...pair, "--concurrency", "foo"])).toThrow(/positive integer/);
+  });
+
+  test("CORBITS_EVAL_CONCURRENCY sets the default", () => {
+    process.env.CORBITS_EVAL_CONCURRENCY = "3";
+    const opts = parseArgs(["--provider", "foo", "--model", "bar"]);
+    expect(opts.concurrency).toBe(3);
+  });
+
+  test("--concurrency overrides CORBITS_EVAL_CONCURRENCY", () => {
+    process.env.CORBITS_EVAL_CONCURRENCY = "8";
+    const opts = parseArgs(["--provider", "foo", "--model", "bar", "--concurrency", "2"]);
+    expect(opts.concurrency).toBe(2);
+  });
+
+  test("invalid CORBITS_EVAL_CONCURRENCY throws", () => {
+    process.env.CORBITS_EVAL_CONCURRENCY = "0";
+    expect(() => parseArgs(["--provider", "foo", "--model", "bar"])).toThrow(
+      /CORBITS_EVAL_CONCURRENCY must be a positive integer/,
+    );
+  });
+});
+
+describe("mapPool", () => {
+  test("N overlapping jobs with concurrency N finish in ~one job duration", async () => {
+    const jobMs = 80;
+    const n = 4;
+    const start = Date.now();
+    const results = await mapPool([0, 1, 2, 3], n, async (item) => {
+      await new Promise((r) => setTimeout(r, jobMs));
+      return item;
+    });
+    const elapsed = Date.now() - start;
+    expect(results).toEqual([0, 1, 2, 3]);
+    expect(elapsed).toBeLessThan(jobMs * 2);
+    expect(elapsed).toBeGreaterThanOrEqual(jobMs - 20);
+  });
+
+  test("preserves input order when later items finish first", async () => {
+    const results = await mapPool([1, 2, 3], 3, async (item) => {
+      await new Promise((r) => setTimeout(r, (4 - item) * 30));
+      return item;
+    });
+    expect(results).toEqual([1, 2, 3]);
+  });
+
+  test("empty input returns an empty array", async () => {
+    expect(await mapPool([], 4, async (item) => item)).toEqual([]);
+  });
+
+  test("rejects non-positive concurrency", async () => {
+    await expect(mapPool([1], 0, async (item) => item)).rejects.toThrow(/positive integer/);
+  });
 });
 
 describe("initEvalGitRepo", () => {

scripts/eval-capability.ts

@@ -68,6 +68,8 @@ type CliOptions = {
   verifyTimeoutMs: number;
   /** Runs per case×variant cell (gate runs use 5; freeze runs use 3). */
   repeats: number;
+  /** Independent case×variant×repeat cells in parallel (default 1). */
+  concurrency: number;
   dryRun: boolean;
   help: boolean;
   /**
@@ -93,18 +95,62 @@ function printUsage(): void {
   --agent-timeout-ms <n> Wall-clock limit for runExec (default 1200000)
   --verify-timeout-ms <n> Wall-clock limit for verify.sh (default 120000)
   --repeats <n>         Runs per case×variant cell (default 1; gate runs use 5)
+  --concurrency <n>     Independent cells in parallel (default 1, env CORBITS_EVAL_CONCURRENCY)
   --dry-run             List cases × variants only (still requires --provider/--model or --matrix)
   --allow-provider-fallback  Allow resolved provider/model to differ from
                              what was requested (default: hard-fail)
   -h, --help            Show help
 `);
 }
 
+function parsePositiveInteger(raw: string, label: string): number {
+  const n = Number(raw);
+  if (!Number.isInteger(n) || n <= 0) {
+    throw new Error(`${label} must be a positive integer`);
+  }
+  return n;
+}
+
+function defaultConcurrency(): number {
+  const raw = process.env.CORBITS_EVAL_CONCURRENCY;
+  if (raw === undefined || raw === "") return 1;
+  return parsePositiveInteger(raw, "CORBITS_EVAL_CONCURRENCY");
+}
+
+/**
+ * Run `mapper` over `items` with at most `concurrency` in flight.
+ * Results stay in input order even when later items finish first.
+ */
+export async function mapPool<T, R>(
+  items: readonly T[],
+  concurrency: number,
+  mapper: (item: T, index: number) => Promise<R>,
+): Promise<R[]> {
+  if (!Number.isInteger(concurrency) || concurrency <= 0) {
+    throw new Error("concurrency must be a positive integer");
+  }
+  if (items.length === 0) return [];
+  const results: R[] = new Array(items.length);
+  let nextIndex = 0;
+  const worker = async (): Promise<void> => {
+    while (true) {
+      const index = nextIndex;
+      nextIndex += 1;
+      if (index >= items.length) return;
+      results[index] = await mapper(items[index]!, index);
+    }
+  };
+  const workerCount = Math.min(concurrency, items.length);
+  await Promise.all(Array.from({ length: workerCount }, () => worker()));
+  return results;
+}
+
 export function parseArgs(argv: readonly string[]): CliOptions {
   const opts: CliOptions = {
     caseSelector: "all",
     skipPermissions: true,
     repeats: 1,
+    concurrency: defaultConcurrency(),
     dryRun: false,
     help: false,
     allowProviderFallback: false,
@@ -175,6 +221,9 @@ export function parseArgs(argv: readonly string[]): CliOptions {
         opts.repeats = n;
         break;
       }
+      case "--concurrency":
+        opts.concurrency = parsePositiveInteger(next(), "--concurrency");
+        break;
       case "--dry-run":
         opts.dryRun = true;
         break;
@@ -771,6 +820,7 @@ async function main(): Promise<number> {
   }
 
   console.log(`Repeats per cell: ${opts.repeats}`);
+  console.log(`Concurrency: ${opts.concurrency}`);
 
   if (opts.dryRun) {
     console.log("dry-run: no inference");
@@ -781,14 +831,15 @@ async function main(): Promise<number> {
   }
 
   const startedAt = new Date().toISOString();
-  const results: CaseResult[] = [];
-
+  const cells: Array<{ caseDef: EvalCase; variant: EvalVariant; repeat: number }> = [];
   for (const { caseDef, variant } of plan) {
     for (let repeat = 0; repeat < opts.repeats; repeat++) {
-      const result = await runCase(caseDef, variant, opts, repeat);
-      results.push(result);
+      cells.push({ caseDef, variant, repeat });
     }
   }
+  const results = await mapPool(cells, opts.concurrency, ({ caseDef, variant, repeat }) =>
+    runCase(caseDef, variant, opts, repeat),
+  );
 
   const finishedAt = new Date().toISOString();
   const totals = summarizeRun(results);