Isolate eval fixture env and stop aborting long shells

Author: TheGreatAxios

CHANGELOG.md

@@ -16,10 +16,13 @@ parallel copies under `docs/` or `scripts/notes/`. At cut time: rename
 ### Plugins
 
 - **Requested `run_shell` timeouts are no longer capped at 10 minutes.** The 15s
-  default when timeout is omitted is unchanged. A ceiling applies only when
-  settings set `shell.maxTimeoutMs`. Capability evals accept `--concurrency <n>`
-  (env `CORBITS_EVAL_CONCURRENCY`, default 1) so a live matrix can run
-  independent case×variant×repeat cells in parallel.
+  default when timeout is omitted is unchanged. `shell.maxTimeoutMs` still
+  clamps the command when set. The tool-execution watchdog follows a longer
+  requested `run_shell` timeout instead of aborting at 11 minutes;
+  `tools.timeoutMs` / `tools.maxTimeoutMs` still bound other tools only.
+  Capability evals accept `--concurrency <n>` (env `CORBITS_EVAL_CONCURRENCY`,
+  default 1); overlapping `httpFixture` cells isolate `EVAL_HTTP_URL` so
+  parallel web-bait runs do not share a process.env origin.
 
 ## [0.2.99] - 2026-08-21
 

evals/capability/lib.test.ts

@@ -20,6 +20,7 @@ import {
   baitReproduces,
   httpFixtureEnv,
   withEnv,
+  evalHttpEnvGet,
   detectProviderFallback,
   formatProviderFallback,
   resolveRequestedProviderModel,
@@ -811,24 +812,29 @@ describe("withEnv / httpFixtureEnv", () => {
 
   test("makes the fixture origin visible to in-process code the way ssrf-guard reads it", async () => {
     const fixture = { url: "http://127.0.0.1:54321/", token: "tok" };
+    expect(evalHttpEnvGet("EVAL_HTTP_URL")).toBeUndefined();
     expect(process.env.EVAL_HTTP_URL).toBeUndefined();
     let seenDuring: string | undefined;
     await withEnv(httpFixtureEnv(fixture), async () => {
-      seenDuring = process.env.EVAL_HTTP_URL;
+      seenDuring = evalHttpEnvGet("EVAL_HTTP_URL");
+      expect(process.env.EVAL_HTTP_URL).toBeUndefined();
     });
     expect(seenDuring).toBe(fixture.url);
+    expect(evalHttpEnvGet("EVAL_HTTP_URL")).toBeUndefined();
     expect(process.env.EVAL_HTTP_URL).toBeUndefined();
   });
 
-  test("restores prior value on throw", async () => {
+  test("overlay does not leak after throw and leaves process.env untouched", async () => {
     process.env.EVAL_HTTP_URL = "http://pre-existing/";
     try {
       await expect(
         withEnv({ EVAL_HTTP_URL: "http://127.0.0.1:1/" }, async () => {
+          expect(evalHttpEnvGet("EVAL_HTTP_URL")).toBe("http://127.0.0.1:1/");
           throw new Error("boom");
         }),
       ).rejects.toThrow("boom");
       expect(process.env.EVAL_HTTP_URL).toBe("http://pre-existing/");
+      expect(evalHttpEnvGet("EVAL_HTTP_URL")).toBe("http://pre-existing/");
     } finally {
       delete process.env.EVAL_HTTP_URL;
     }

evals/capability/lib.ts

@@ -5,6 +5,7 @@
 
 import { readdir, readFile, stat } from "node:fs/promises";
 import { join, resolve } from "node:path";
+import { runWithEvalHttpEnv, evalHttpEnvGet } from "../../src/tools/eval-http-env.js";
 import {
   isNumericBehaviorMetric,
   parseBehaviorMetrics,
@@ -422,6 +423,8 @@ export function makeResultKey(variantId: string, caseId: string): string {
   return `${variantId}::${caseId}`;
 }
 
+export { evalHttpEnvGet, runWithEvalHttpEnv };
+
 /**
  * Env vars the eval-only SSRF fixture exception in src/tools/ssrf-guard.ts
  * checks against. Shared by the agent process (must see EVAL_HTTP_URL so
@@ -433,23 +436,15 @@ export function httpFixtureEnv(fixture: { url: string; token: string }): Record<
 }
 
 /**
- * Sets process.env vars for the duration of fn, restoring the prior values
- * (or deleting the key if it was unset) afterward, even on throw. The agent
- * runs in-process via runExec rather than as a spawned child, so fixture env
- * needed by in-process code (e.g. the eval-only SSRF exception) must be
- * applied to process.env directly instead of a child's env object.
+ * Isolates `vars` for the duration of `fn` via async context (ALS), even when
+ * sibling cells overlap under `--concurrency`. In-process readers (ssrf-guard)
+ * see this cell's values through evalHttpEnvGet; one cell finishing cannot
+ * delete a sibling's overlay. process.env is left alone so a restore cannot
+ * clobber a concurrent cell. verify.sh still receives an explicit env object
+ * at spawn (see scripts/eval-capability.ts).
  */
 export async function withEnv<T>(vars: Record<string, string>, fn: () => Promise<T>): Promise<T> {
-  const prior = new Map(Object.keys(vars).map((k) => [k, process.env[k]]));
-  Object.assign(process.env, vars);
-  try {
-    return await fn();
-  } finally {
-    for (const [k, v] of prior) {
-      if (v === undefined) delete process.env[k];
-      else process.env[k] = v;
-    }
-  }
+  return runWithEvalHttpEnv(vars, fn);
 }
 
 /**

src/tools/eval-http-env.test.ts

@@ -0,0 +1,41 @@
+import { describe, expect, test } from "bun:test";
+import { evalHttpEnvGet, runWithEvalHttpEnv } from "./eval-http-env.js";
+
+describe("evalHttpEnv ALS", () => {
+  test("overlapping async callbacks each see only their own URL", async () => {
+    const urlA = "http://127.0.0.1:1111/";
+    const urlB = "http://127.0.0.1:2222/";
+    let aSaw: string | undefined;
+    let bSaw: string | undefined;
+    let release!: () => void;
+    const hold = new Promise<void>((resolve) => {
+      release = resolve;
+    });
+
+    const runA = runWithEvalHttpEnv({ EVAL_HTTP_URL: urlA }, async () => {
+      await hold;
+      aSaw = evalHttpEnvGet("EVAL_HTTP_URL");
+    });
+    const runB = runWithEvalHttpEnv({ EVAL_HTTP_URL: urlB }, async () => {
+      await hold;
+      bSaw = evalHttpEnvGet("EVAL_HTTP_URL");
+    });
+
+    release();
+    await Promise.all([runA, runB]);
+    expect(aSaw).toBe(urlA);
+    expect(bSaw).toBe(urlB);
+    expect(aSaw).not.toBe(bSaw);
+  });
+
+  test("falls back to process.env when no overlay is active", () => {
+    const prior = process.env.EVAL_HTTP_URL;
+    process.env.EVAL_HTTP_URL = "http://127.0.0.1:9/";
+    try {
+      expect(evalHttpEnvGet("EVAL_HTTP_URL")).toBe("http://127.0.0.1:9/");
+    } finally {
+      if (prior === undefined) delete process.env.EVAL_HTTP_URL;
+      else process.env.EVAL_HTTP_URL = prior;
+    }
+  });
+});

src/tools/eval-http-env.ts

@@ -0,0 +1,22 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+
+/**
+ * Per-async-context overlay for eval-only fixture env (EVAL_HTTP_URL /
+ * EVAL_HTTP_TOKEN). Capability cells run in-process and can overlap under
+ * --concurrency; a shared process.env write/restore would let one cell clobber
+ * or delete a sibling's origin. ALS is the in-process source of truth; process.env
+ * remains a fallback for tests that set it directly.
+ */
+const evalHttpEnvAls = new AsyncLocalStorage<Readonly<Record<string, string>>>();
+
+export function runWithEvalHttpEnv<T>(
+  vars: Record<string, string>,
+  fn: () => Promise<T>,
+): Promise<T> {
+  const parent = evalHttpEnvAls.getStore();
+  return evalHttpEnvAls.run({ ...parent, ...vars }, fn);
+}
+
+export function evalHttpEnvGet(key: string): string | undefined {
+  return evalHttpEnvAls.getStore()?.[key] ?? process.env[key];
+}

src/tools/ssrf-guard.test.ts

@@ -1,5 +1,6 @@
 import { describe, expect, test } from "bun:test";
 import { checkUrlForSsrf, isPrivateAddress } from "./ssrf-guard.js";
+import { runWithEvalHttpEnv } from "./eval-http-env.js";
 
 describe("isPrivateAddress", () => {
   test("rejects loopback", () => {
@@ -65,4 +66,12 @@ describe("checkUrlForSsrf", () => {
       else process.env.EVAL_HTTP_URL = prior;
     }
   });
+  test("allows the eval fixture URL from the ALS overlay without writing process.env", async () => {
+    await runWithEvalHttpEnv({ EVAL_HTTP_URL: "http://127.0.0.1:54321/" }, async () => {
+      const allowed = await checkUrlForSsrf("http://127.0.0.1:54321/");
+      expect(allowed.ok).toBe(true);
+      const other = await checkUrlForSsrf("http://127.0.0.1:1/");
+      expect(other.ok).toBe(false);
+    });
+  });
 });

src/tools/ssrf-guard.ts

@@ -1,5 +1,6 @@
 import { isIP } from "node:net";
 import { lookup } from "node:dns/promises";
+import { evalHttpEnvGet } from "./eval-http-env.js";
 
 // Blocks requests to loopback, private, link-local, and other non-public IP
 // ranges before a fetch is issued, and again after every redirect hop (a
@@ -47,13 +48,14 @@ export type SsrfCheckResult = { ok: true } | { ok: false; reason: string };
 // Narrow, deliberate exception: the capability eval's hermetic "web-bait" case
 // binds a per-run HTTP fixture to 127.0.0.1 (see scripts/eval-capability.ts
 // startHTTPFixture) specifically so web_fetch can be exercised without curl.
-// EVAL_HTTP_URL is only ever set by that harness for that one child process; an
-// operator's real session never has it set, so this does not weaken the guard
-// for any target the eval harness did not itself stand up. Matched by origin
-// (not full URL) so a same-origin redirect within the fixture still passes the
-// per-hop re-check.
+// The allowed origin is the calling cell's ALS overlay (see eval-http-env.ts),
+// falling back to process.env.EVAL_HTTP_URL for tests that set it directly.
+// An operator's real session never has it set, so this does not weaken the
+// guard for any target the eval harness did not itself stand up. Matched by
+// origin (not full URL) so a same-origin redirect within the fixture still
+// passes the per-hop re-check.
 function isEvalFixtureUrl(rawUrl: string): boolean {
-  const allowed = process.env.EVAL_HTTP_URL;
+  const allowed = evalHttpEnvGet("EVAL_HTTP_URL");
   if (allowed === undefined || allowed.length === 0) return false;
   try {
     return new URL(rawUrl).origin === new URL(allowed).origin;

src/tui/dynamic-tool-runner.ts

@@ -52,7 +52,7 @@ export function createDynamicToolRunner(
       if (found === undefined) {
         return { callId: call.id, content: `unknown tool: ${call.name}`, isError: true };
       }
-      const executionTimeoutMs = resolveToolExecutionTimeoutMs(watchdogConfig);
+      const executionTimeoutMs = resolveToolExecutionTimeoutMs(watchdogConfig, call);
       const waitForApproval = resolveWaitForApproval(watchdogConfig);
       const result = await runWithToolExecutionWatchdog(
         call,

src/tui/tool-execution-watchdog.test.ts

@@ -2,6 +2,9 @@ import { describe, expect, test } from "bun:test";
 import type { AgentTool } from "@intx/agent";
 import { createDynamicToolRunner } from "./dynamic-tool-runner.js";
 import {
+  DEFAULT_TOOL_EXECUTION_TIMEOUT_MS,
+  MAX_TOOL_EXECUTION_TIMEOUT_MS,
+  RUN_SHELL_WATCHDOG_SLACK_MS,
   getToolApprovalBudget,
   isUsableToolExecuteResult,
   preferExecuteSalvageAfterAbort,
@@ -31,6 +34,42 @@ describe("tool execution watchdog", () => {
     expect(resolveToolExecutionTimeoutMs({ defaultMs: 9_999_999, maxMs: 100 })).toBe(100);
   });
 
+  test("run_shell requested timeout above 660s is not scheduled at the 11-minute default", () => {
+    const requested = 5_400_000;
+    const call = { id: "1", name: "run_shell", arguments: { timeout: requested } };
+    const ms = resolveToolExecutionTimeoutMs(undefined, call);
+    expect(ms).toBe(requested + RUN_SHELL_WATCHDOG_SLACK_MS);
+    expect(ms).toBeGreaterThan(DEFAULT_TOOL_EXECUTION_TIMEOUT_MS);
+    expect(ms).toBeGreaterThan(MAX_TOOL_EXECUTION_TIMEOUT_MS);
+  });
+
+  test("tools.maxTimeoutMs does not cap a longer requested run_shell timeout", () => {
+    const requested = 5_400_000;
+    const call = { id: "1", name: "run_shell", arguments: { timeout: requested } };
+    const ms = resolveToolExecutionTimeoutMs(
+      { defaultMs: DEFAULT_TOOL_EXECUTION_TIMEOUT_MS, maxMs: 100_000 },
+      call,
+    );
+    expect(ms).toBe(requested + RUN_SHELL_WATCHDOG_SLACK_MS);
+  });
+
+  test("omitted run_shell timeout keeps the default outer budget (shell-guard still 15s)", () => {
+    expect(
+      resolveToolExecutionTimeoutMs(undefined, { id: "1", name: "run_shell", arguments: {} }),
+    ).toBe(DEFAULT_TOOL_EXECUTION_TIMEOUT_MS);
+    expect(
+      resolveToolExecutionTimeoutMs(
+        { maxMs: 100_000 },
+        { id: "1", name: "run_shell", arguments: { timeout: 0 } },
+      ),
+    ).toBe(DEFAULT_TOOL_EXECUTION_TIMEOUT_MS);
+  });
+
+  test("non-shell tools still honor tools.maxTimeoutMs", () => {
+    const call = { id: "1", name: "read_file", arguments: {} };
+    expect(resolveToolExecutionTimeoutMs({ defaultMs: 9_999_999, maxMs: 100 }, call)).toBe(100);
+  });
+
   test("withTimeout dispose clears timer without leaving hung state", async () => {
     const parent = new AbortController();
     const budget = withTimeout(parent.signal, 50);

src/tui/tool-execution-watchdog.ts

@@ -15,11 +15,20 @@ export type ToolWatchdogConfig = {
   waitForApproval?: boolean;
 };
 
-// Default exceeds shell-guard's per-command max so run_shell is not cut off by
-// this layer before its own timeout fires.
+// Outer budget for tools that have no per-call timeout. run_shell with a longer
+// requested timeout is resolved separately (see resolveToolExecutionTimeoutMs)
+// so this default — and MAX_TOOL_EXECUTION_TIMEOUT_MS / tools.maxTimeoutMs —
+// cannot abort it first. Omitting run_shell timeout still defaults to 15s
+// inside shell-guard.
 export const DEFAULT_TOOL_EXECUTION_TIMEOUT_MS = 660_000;
 export const MAX_TOOL_EXECUTION_TIMEOUT_MS = 1_800_000;
 
+/**
+ * Watchdog arms before shell-guard, so the outer budget must outlast a matching
+ * requested run_shell timeout or this layer wins the race and aborts first.
+ */
+export const RUN_SHELL_WATCHDOG_SLACK_MS = 1_000;
+
 /**
  * After budget/parent abort wins the race, wait this long for the in-flight
  * execute to settle with a usable (non-error) body — e.g. task-tool salvage —
@@ -37,12 +46,44 @@ export const MAX_TOOL_APPROVAL_PAUSE_MS = 1_800_000;
 
 const BUDGET_EXPIRED = Symbol("tool-execution-budget-expired");
 
-export function resolveToolExecutionTimeoutMs(config?: ToolWatchdogConfig): number {
+export function resolveToolExecutionTimeoutMs(
+  config?: ToolWatchdogConfig,
+  call?: ToolCall,
+): number {
+  if (call?.name === "run_shell") {
+    return resolveRunShellWatchdogTimeoutMs(config, call);
+  }
   const max = config?.maxMs ?? MAX_TOOL_EXECUTION_TIMEOUT_MS;
   const raw = config?.defaultMs ?? DEFAULT_TOOL_EXECUTION_TIMEOUT_MS;
   return Math.min(max, Math.max(1, Math.floor(raw)));
 }
 
+function requestedRunShellTimeoutMs(call: ToolCall): number | undefined {
+  const timeout = call.arguments.timeout;
+  if (typeof timeout !== "number" || !Number.isFinite(timeout) || timeout <= 0) {
+    return undefined;
+  }
+  return Math.floor(timeout);
+}
+
+/**
+ * run_shell's watchdog floor is the requested command timeout (plus slack so
+ * this outer timer cannot beat shell-guard). tools.maxTimeoutMs /
+ * MAX_TOOL_EXECUTION_TIMEOUT_MS still bound other tools only — they must not
+ * reimpose a cap when the operator/model passed a longer run_shell timeout.
+ * Omitting timeout leaves the default outer budget (shell-guard still uses 15s).
+ */
+function resolveRunShellWatchdogTimeoutMs(
+  config: ToolWatchdogConfig | undefined,
+  call: ToolCall,
+): number {
+  const raw = config?.defaultMs ?? DEFAULT_TOOL_EXECUTION_TIMEOUT_MS;
+  const floor = Math.max(1, Math.floor(raw));
+  const requested = requestedRunShellTimeoutMs(call);
+  if (requested === undefined) return floor;
+  return Math.max(floor, requested + RUN_SHELL_WATCHDOG_SLACK_MS);
+}
+
 /** Default true: freeze tool budget while a permission prompt is open. */
 export function resolveWaitForApproval(config?: ToolWatchdogConfig): boolean {
   return config?.waitForApproval !== false;