From 867bea137dd3c33ee3e7bfc035f887e11a3fbffa Mon Sep 17 00:00:00 2001 From: hasak21 Date: Sat, 29 Aug 2026 18:55:11 +0800 Subject: [PATCH 1/3] fix(workflows): enforce sandbox execution timeout across async continuations (#285) --- extensions/workflows/sandbox.ts | 29 ++++++++++++++++++++++ tests/extensions/workflows/sandbox.test.ts | 16 ++++++++++++ 2 files changed, 45 insertions(+) diff --git a/extensions/workflows/sandbox.ts b/extensions/workflows/sandbox.ts index 5aa5bfd4..53833c1d 100644 --- a/extensions/workflows/sandbox.ts +++ b/extensions/workflows/sandbox.ts @@ -19,6 +19,11 @@ const MAX_LOG_MESSAGE_BYTES = 8 * 1024; * bypasses the controller entirely. */ export const AGENT_CALL_BACKSTOP_MARGIN = 8; +/** + * Maximum synchronous execution time allowed between async agent boundaries. + * Non-yielding code (e.g. while(true){}) after await is terminated by this timeout. + */ +export const SANDBOX_SYNC_TIMEOUT_MS = 1_000; export interface SandboxAgentOptions { agent_type?: unknown; @@ -192,6 +197,24 @@ export function runWorkflowSandbox(options: RunWorkflowSandboxOptions) { const activeAgentRequests = new Map(); let requestCount = 0; let finished = false; + let executionWatchdog: NodeJS.Timeout | undefined; + + const armWatchdog = (timeoutMs = SANDBOX_SYNC_TIMEOUT_MS) => { + if (finished) return; + if (executionWatchdog) clearTimeout(executionWatchdog); + executionWatchdog = setTimeout(() => { + if (finished) return; + finish(new Error("Script execution timed out")); + }, timeoutMs); + executionWatchdog.unref?.(); + }; + + const disarmWatchdog = () => { + if (executionWatchdog) { + clearTimeout(executionWatchdog); + executionWatchdog = undefined; + } + }; // The child parses this and falls back to zeros if it is ever unusable, so // a broken snapshot degrades `usage()` to a zero reading instead of @@ -205,6 +228,7 @@ export function runWorkflowSandbox(options: RunWorkflowSandboxOptions) { }; const cleanup = () => { + disarmWatchdog(); for (const abortController of activeAgentRequests.values()) { abortController.abort(new Error("Workflow stopped")); } @@ -294,6 +318,7 @@ export function runWorkflowSandbox(options: RunWorkflowSandboxOptions) { return; } if (raw.kind === "agent") { + disarmWatchdog(); if ( typeof raw.payloadJson !== "string" || byteLength(raw.payloadJson) > MAX_AGENT_MESSAGE_BYTES @@ -352,6 +377,9 @@ export function runWorkflowSandbox(options: RunWorkflowSandboxOptions) { resultJson, usageJson: usageJson(), }); + if (activeAgentRequests.size === 0) { + armWatchdog(); + } }; activeAgentRequests.set(id, abortController); let agentOperation: Promise; @@ -406,6 +434,7 @@ export function runWorkflowSandbox(options: RunWorkflowSandboxOptions) { }, (error) => { if (error) finish(error); + else armWatchdog(); }, ); }); diff --git a/tests/extensions/workflows/sandbox.test.ts b/tests/extensions/workflows/sandbox.test.ts index 0311e5fa..02c35bf4 100644 --- a/tests/extensions/workflows/sandbox.test.ts +++ b/tests/extensions/workflows/sandbox.test.ts @@ -204,6 +204,22 @@ test("sandbox VM still rejects non-yielding synchronous code", async () => { await assert.rejects(run(`while (true) {}`), /timed out/); }); +test("sandbox rejects non-yielding synchronous code after await Promise.resolve()", async () => { + await assert.rejects( + run(`await Promise.resolve(); while (true) {}`), + /timed out/, + ); +}); + +test("sandbox rejects non-yielding synchronous code after await agent()", async () => { + await assert.rejects( + run(`await agent("step"); while (true) {}`, { + onAgent: async () => ({ ok: true, output: "done" }), + }), + /timed out/, + ); +}); + test("workflow sandbox imposes no fixed whole-agent wall timer", async () => { let signalAborted = false; const result = await run(`return (await agent("delayed")).output;`, { From 8f4d30b39d045a190ab6bc07c9cd61086ff5c734 Mon Sep 17 00:00:00 2001 From: hasak21 Date: Sat, 29 Aug 2026 21:07:19 +0800 Subject: [PATCH 2/3] fix(workflows): explicitly arm watchdog on init and expand async boundary tests --- extensions/workflows/sandbox.ts | 5 +++- tests/extensions/workflows/sandbox.test.ts | 35 ++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/extensions/workflows/sandbox.ts b/extensions/workflows/sandbox.ts index 53833c1d..99efde1c 100644 --- a/extensions/workflows/sandbox.ts +++ b/extensions/workflows/sandbox.ts @@ -423,6 +423,10 @@ export function runWorkflowSandbox(options: RunWorkflowSandboxOptions) { finish(new Error("Workflow sandbox sent an unknown IPC message")); }); + // Arm the synchronous execution watchdog for the initial script invocation + // (covers non-yielding code before and after initial microtask yields). + armWatchdog(); + child.send( { kind: "init", @@ -434,7 +438,6 @@ export function runWorkflowSandbox(options: RunWorkflowSandboxOptions) { }, (error) => { if (error) finish(error); - else armWatchdog(); }, ); }); diff --git a/tests/extensions/workflows/sandbox.test.ts b/tests/extensions/workflows/sandbox.test.ts index 02c35bf4..08e19502 100644 --- a/tests/extensions/workflows/sandbox.test.ts +++ b/tests/extensions/workflows/sandbox.test.ts @@ -211,6 +211,15 @@ test("sandbox rejects non-yielding synchronous code after await Promise.resolve( ); }); +test("sandbox rejects non-yielding synchronous code after chained microtasks", async () => { + await assert.rejects( + run( + `await Promise.resolve().then(() => Promise.resolve()); while (true) {}`, + ), + /timed out/, + ); +}); + test("sandbox rejects non-yielding synchronous code after await agent()", async () => { await assert.rejects( run(`await agent("step"); while (true) {}`, { @@ -220,6 +229,32 @@ test("sandbox rejects non-yielding synchronous code after await agent()", async ); }); +test("sandbox rejects non-yielding synchronous code between sequential agent calls", async () => { + let firstCallSettled = false; + await assert.rejects( + run(`await agent("first"); while (true) {}; await agent("second");`, { + onAgent: async () => { + firstCallSettled = true; + return { ok: true, output: "first-done" }; + }, + }), + /timed out/, + ); + assert.equal(firstCallSettled, true); +}); + +test("sandbox rejects non-yielding synchronous code after parallel agent calls", async () => { + await assert.rejects( + run( + `await parallel([() => agent("p1"), () => agent("p2")]); while (true) {}`, + { + onAgent: async (prompt) => ({ ok: true, output: `${prompt}-done` }), + }, + ), + /timed out/, + ); +}); + test("workflow sandbox imposes no fixed whole-agent wall timer", async () => { let signalAborted = false; const result = await run(`return (await agent("delayed")).output;`, { From dd8c8bd4646be18366613a2ea80a9a7abc7c6da4 Mon Sep 17 00:00:00 2001 From: hasak21 Date: Mon, 31 Aug 2026 10:55:07 +0800 Subject: [PATCH 3/3] fix(workflows): track execution windows across interleaved continuations and add test deadlines (#285) --- extensions/workflows/sandbox-child.cjs | 16 +++- extensions/workflows/sandbox.ts | 27 ++++--- tests/extensions/workflows/sandbox.test.ts | 94 +++++++++++++++++++--- 3 files changed, 116 insertions(+), 21 deletions(-) diff --git a/extensions/workflows/sandbox-child.cjs b/extensions/workflows/sandbox-child.cjs index 30124c66..ba4fe3c3 100644 --- a/extensions/workflows/sandbox-child.cjs +++ b/extensions/workflows/sandbox-child.cjs @@ -309,8 +309,9 @@ process.on("message", (message) => { } initialized = true; token = message.token; + const seq = typeof message.seq === "number" ? message.seq : undefined; if (typeof message.usageJson === "string") usageJson = message.usageJson; - run(message.source, message.argsJson, message.maxConcurrency); + run(message.source, message.argsJson, message.maxConcurrency, seq); return; } if (message.token !== token || message.kind !== "agentResult") return; @@ -318,6 +319,7 @@ process.on("message", (message) => { const pending = pendingAgents.get(message.id); if (!pending) return; pendingAgents.delete(message.id); + const seq = typeof message.seq === "number" ? message.seq : undefined; if (typeof message.resultJson === "string") pending.resolve(message.resultJson); else @@ -326,9 +328,14 @@ process.on("message", (message) => { typeof message.error === "string" ? message.error : "Agent IPC failed", ), ); + if (seq !== undefined) { + setImmediate(() => { + send({ kind: "idle", seq }); + }); + } }); -function run(source, argsJson, maxConcurrency) { +function run(source, argsJson, maxConcurrency, initSeq) { try { const sandbox = Object.create(null); sandbox.__argsJson = argsJson; @@ -400,6 +407,11 @@ function run(source, argsJson, maxConcurrency) { context, { timeout: 1000 }, ); + if (initSeq !== undefined) { + setImmediate(() => { + send({ kind: "idle", seq: initSeq }); + }); + } Promise.resolve(context.__workflowPromise) .then((resultJson) => { if (typeof resultJson !== "string") diff --git a/extensions/workflows/sandbox.ts b/extensions/workflows/sandbox.ts index 99efde1c..08005763 100644 --- a/extensions/workflows/sandbox.ts +++ b/extensions/workflows/sandbox.ts @@ -23,7 +23,7 @@ export const AGENT_CALL_BACKSTOP_MARGIN = 8; * Maximum synchronous execution time allowed between async agent boundaries. * Non-yielding code (e.g. while(true){}) after await is terminated by this timeout. */ -export const SANDBOX_SYNC_TIMEOUT_MS = 1_000; +const SANDBOX_SYNC_TIMEOUT_MS = 1_000; export interface SandboxAgentOptions { agent_type?: unknown; @@ -197,19 +197,23 @@ export function runWorkflowSandbox(options: RunWorkflowSandboxOptions) { const activeAgentRequests = new Map(); let requestCount = 0; let finished = false; + let executionSeq = 0; let executionWatchdog: NodeJS.Timeout | undefined; const armWatchdog = (timeoutMs = SANDBOX_SYNC_TIMEOUT_MS) => { - if (finished) return; + if (finished) return 0; if (executionWatchdog) clearTimeout(executionWatchdog); + const currentSeq = ++executionSeq; executionWatchdog = setTimeout(() => { - if (finished) return; + if (finished || currentSeq !== executionSeq) return; finish(new Error("Script execution timed out")); }, timeoutMs); executionWatchdog.unref?.(); + return currentSeq; }; - const disarmWatchdog = () => { + const disarmWatchdog = (seq?: number) => { + if (seq !== undefined && seq !== executionSeq) return; if (executionWatchdog) { clearTimeout(executionWatchdog); executionWatchdog = undefined; @@ -317,8 +321,13 @@ export function runWorkflowSandbox(options: RunWorkflowSandboxOptions) { } return; } + if (raw.kind === "idle") { + if (typeof raw.seq === "number") { + disarmWatchdog(raw.seq); + } + return; + } if (raw.kind === "agent") { - disarmWatchdog(); if ( typeof raw.payloadJson !== "string" || byteLength(raw.payloadJson) > MAX_AGENT_MESSAGE_BYTES @@ -370,16 +379,15 @@ export function runWorkflowSandbox(options: RunWorkflowSandboxOptions) { error: "Agent result exceeded the workflow IPC output limit", }); } + const seq = armWatchdog(); child.send({ token, kind: "agentResult", id, + seq, resultJson, usageJson: usageJson(), }); - if (activeAgentRequests.size === 0) { - armWatchdog(); - } }; activeAgentRequests.set(id, abortController); let agentOperation: Promise; @@ -425,7 +433,7 @@ export function runWorkflowSandbox(options: RunWorkflowSandboxOptions) { // Arm the synchronous execution watchdog for the initial script invocation // (covers non-yielding code before and after initial microtask yields). - armWatchdog(); + const initSeq = armWatchdog(); child.send( { @@ -435,6 +443,7 @@ export function runWorkflowSandbox(options: RunWorkflowSandboxOptions) { argsJson, maxConcurrency: options.maxConcurrency, usageJson: usageJson(), + seq: initSeq, }, (error) => { if (error) finish(error); diff --git a/tests/extensions/workflows/sandbox.test.ts b/tests/extensions/workflows/sandbox.test.ts index 08e19502..673dac1d 100644 --- a/tests/extensions/workflows/sandbox.test.ts +++ b/tests/extensions/workflows/sandbox.test.ts @@ -25,6 +25,36 @@ function run( }); } +async function runWithTimeout( + source: string, + overrides: Partial[0]> = {}, + timeoutMs = 4000, +) { + const abort = new AbortController(); + const timer = setTimeout(() => { + abort.abort(new Error(`Test deadline of ${timeoutMs}ms exceeded`)); + }, timeoutMs); + timer.unref?.(); + try { + return await runWorkflowSandbox({ + source, + args: undefined, + cwd: process.cwd(), + signal: overrides.signal ?? abort.signal, + onAgent: async (prompt) => ({ ok: true, output: `reply:${prompt}` }), + onPhase: () => {}, + onLog: () => {}, + usageSnapshot: () => ({ total: 0 }), + maxConcurrency: 8, + maxAgentCalls: 128, + ...overrides, + }); + } finally { + clearTimeout(timer); + abort.abort(); + } +} + test("sandbox exposes only workflow capabilities and validates results", async () => { const phases: string[] = []; let active = 0; @@ -201,19 +231,19 @@ test("sandbox source cannot escape the host accounting wrapper", async () => { }); test("sandbox VM still rejects non-yielding synchronous code", async () => { - await assert.rejects(run(`while (true) {}`), /timed out/); + await assert.rejects(runWithTimeout(`while (true) {}`), /timed out/); }); test("sandbox rejects non-yielding synchronous code after await Promise.resolve()", async () => { await assert.rejects( - run(`await Promise.resolve(); while (true) {}`), + runWithTimeout(`await Promise.resolve(); while (true) {}`), /timed out/, ); }); test("sandbox rejects non-yielding synchronous code after chained microtasks", async () => { await assert.rejects( - run( + runWithTimeout( `await Promise.resolve().then(() => Promise.resolve()); while (true) {}`, ), /timed out/, @@ -222,7 +252,7 @@ test("sandbox rejects non-yielding synchronous code after chained microtasks", a test("sandbox rejects non-yielding synchronous code after await agent()", async () => { await assert.rejects( - run(`await agent("step"); while (true) {}`, { + runWithTimeout(`await agent("step"); while (true) {}`, { onAgent: async () => ({ ok: true, output: "done" }), }), /timed out/, @@ -232,12 +262,15 @@ test("sandbox rejects non-yielding synchronous code after await agent()", async test("sandbox rejects non-yielding synchronous code between sequential agent calls", async () => { let firstCallSettled = false; await assert.rejects( - run(`await agent("first"); while (true) {}; await agent("second");`, { - onAgent: async () => { - firstCallSettled = true; - return { ok: true, output: "first-done" }; + runWithTimeout( + `await agent("first"); while (true) {}; await agent("second");`, + { + onAgent: async () => { + firstCallSettled = true; + return { ok: true, output: "first-done" }; + }, }, - }), + ), /timed out/, ); assert.equal(firstCallSettled, true); @@ -245,7 +278,7 @@ test("sandbox rejects non-yielding synchronous code between sequential agent cal test("sandbox rejects non-yielding synchronous code after parallel agent calls", async () => { await assert.rejects( - run( + runWithTimeout( `await parallel([() => agent("p1"), () => agent("p2")]); while (true) {}`, { onAgent: async (prompt) => ({ ok: true, output: `${prompt}-done` }), @@ -255,6 +288,47 @@ test("sandbox rejects non-yielding synchronous code after parallel agent calls", ); }); +test("sandbox rejects non-yielding synchronous code in interleaved parallel branch while another agent is pending", async () => { + let fastCalled = false; + let slowCalled = false; + + await assert.rejects( + runWithTimeout( + ` + await parallel([ + async () => { + await agent("fast"); + while (true) {} + }, + async () => { + await agent("slow"); + return "slow-done"; + }, + ]); + `, + { + onAgent: async (prompt) => { + if (prompt === "fast") { + fastCalled = true; + return { ok: true, output: "fast-ok" }; + } + if (prompt === "slow") { + slowCalled = true; + await new Promise((resolve) => setTimeout(resolve, 2000)); + return { ok: true, output: "slow-ok" }; + } + return { ok: true, output: "unknown" }; + }, + }, + 4000, + ), + /timed out/, + ); + + assert.equal(fastCalled, true); + assert.equal(slowCalled, true); +}); + test("workflow sandbox imposes no fixed whole-agent wall timer", async () => { let signalAborted = false; const result = await run(`return (await agent("delayed")).output;`, {