End a request's sandbox fibers before its database connection closes

apps/cloud/src/api/protected.test.ts

@@ -20,6 +20,8 @@ const makeBaseEngine = (): ExecutionEngine =>
     pausedExecutionCount: () => Effect.succeed(0),
     hasPausedExecutions: () => Effect.succeed(false),
     getDescription: Effect.succeed("desc"),
+    // The fake forks nothing, so there is no sandbox fiber to end.
+    shutdown: Effect.void,
   }) as ExecutionEngine;
 
 describe("withExecutionUsageTracking", () => {

apps/cloud/src/engine/execution-gate.ts

@@ -108,6 +108,9 @@ export const withPreExecutionGate = <E extends Cause.YieldableError>(
   pausedExecutionCount: () => engine.pausedExecutionCount(),
   hasPausedExecutions: () => engine.hasPausedExecutions(),
   getDescription: engine.getDescription,
+  // Forwarded, not re-implemented: the wrapped engine owns the sandbox fibers,
+  // so the host's request-scope teardown has to reach through this decorator.
+  shutdown: engine.shutdown,
 });
 
 // ---------------------------------------------------------------------------

apps/cloud/src/engine/execution-rate-limit.node.test.ts

@@ -31,6 +31,8 @@ const engineStub: ExecutionEngine = {
   pausedExecutionCount: () => Effect.succeed(0),
   hasPausedExecutions: () => Effect.succeed(false),
   getDescription: Effect.succeed("stub"),
+  // The stub forks nothing, so there is no sandbox fiber to end.
+  shutdown: Effect.void,
 };
 
 /** Counter that hands out a caller-controlled sequence of counts. */

apps/cloud/src/engine/execution-usage.ts

@@ -25,4 +25,7 @@ export const withExecutionUsageTracking = <E extends Cause.YieldableError>(
   pausedExecutionCount: () => engine.pausedExecutionCount(),
   hasPausedExecutions: () => engine.hasPausedExecutions(),
   getDescription: engine.getDescription,
+  // Forwarded, not re-implemented: the wrapped engine owns the sandbox fibers,
+  // so the host's request-scope teardown has to reach through this decorator.
+  shutdown: engine.shutdown,
 });

packages/core/analytics/src/engine.test.ts

@@ -60,6 +60,8 @@ const makeFakeEngine = (
   pausedExecutionCount: () => Effect.succeed(0),
   hasPausedExecutions: () => Effect.succeed(false),
   getDescription: Effect.succeed("fake"),
+  // The fake forks nothing, so there is no sandbox fiber to end.
+  shutdown: Effect.void,
 });
 
 describe("withExecutionAnalytics", () => {

packages/core/api/src/server/execution-stack-middleware.ts

@@ -259,6 +259,25 @@ export const makeExecutionStackMiddleware = <
             Effect.provideService(ExecutorService, executor),
             Effect.provideService(ExecutionEngineService, engine),
             provideExecutorExtensions(executor),
+            // This engine belongs to THIS request: the stack above was built
+            // from the host's request-scoped DB handle, and `executeWithPause`
+            // forks its sandbox as a daemon that would otherwise outlive the
+            // handler. The host closes the connection when the request scope
+            // closes — which happens AFTER this effect returns — so ending the
+            // engine here is what keeps a sandbox fiber from waking up on a
+            // closed pool.
+            //
+            // Nothing is lost by ending it: on a per-request engine the paused
+            // fiber is already unreachable once the response is written (a
+            // resume lands on a different engine and replays the call instead),
+            // so the fiber could only ever have failed. The MCP session Durable
+            // Object does NOT come through here — it builds its own stack over a
+            // session-lifetime handle, so its pauses still survive between
+            // requests, which is the whole point of that plane.
+            //
+            // `ensuring`, not `tap`: interruption and failure have to end the
+            // fiber too, and it must not run before the response is produced.
+            Effect.ensuring(engine.shutdown),
           );
           // Provide the boot-captured context; uncaptured deps (cloud's
           // request-scoped `DbService`) remain residual and flow through here.
@@ -315,6 +334,9 @@ const readOnlyExecutionEngine: ExecutionEngine<Cause.YieldableError> = {
   getPausedExecution: () => Effect.succeed(null),
   pausedExecutionCount: () => Effect.succeed(0),
   hasPausedExecutions: () => Effect.succeed(false),
+  // Nothing is ever forked here — the platform branch cannot execute — so there
+  // is no sandbox fiber to end.
+  shutdown: Effect.void,
   // oxlint-disable-next-line executor/no-effect-escape-hatch -- boundary: only the MCP tool server reads this, and the MCP plane never serves a platform credential
   getDescription: Effect.die(new PlatformEngineUnavailable({ member: "getDescription" })),
 };

packages/core/execution/src/engine.ts

@@ -484,13 +484,39 @@ export type ExecutionEngine<E extends Cause.YieldableError = CodeExecutionError>
    * Get the dynamic tool description (workflow + namespaces).
    */
   readonly getDescription: Effect.Effect<string>;
+
+  /**
+   * End this engine's in-flight sandbox fibers and wait for them to unwind.
+   *
+   * `executeWithPause` forks the sandbox as a daemon so a pause can outlive the
+   * caller that observed it. That fiber holds the executor, and so the DB handle
+   * belonging to whichever scope built this engine. A host that builds an engine
+   * per request MUST call this before that scope's connection is closed, or the
+   * fiber outlives the pool it queries.
+   *
+   * Required, not optional: a decorator that wrapped the engine and quietly
+   * dropped this member would silently reopen that race, so the type makes
+   * forwarding it a compile error.
+   */
+  readonly shutdown: Effect.Effect<void>;
 };
 
 export const createExecutionEngine = <E extends Cause.YieldableError = CodeExecutionError>(
   config: ExecutionEngineConfig<E>,
 ): ExecutionEngine<E> => {
   const { executor, codeExecutor, toolDiscoveryProvider = defaultToolDiscoveryProvider } = config;
   const pausedExecutions = new Map<string, InternalPausedExecution<E>>();
+  // Every sandbox fiber `startPausableExecution` still has in flight.
+  //
+  // Those fibers are daemons (`Effect.forkDetach`) so a pause can outlive the
+  // caller that observed it. But they close over `executor`, and the executor
+  // closes over the FumaDB handle the host opened for whatever scope built THIS
+  // engine — `makeFumaClient` captures `db` at construction, not per operation.
+  // A host that builds one engine per HTTP request therefore needs a way to end
+  // that fiber's life with the request; otherwise it wakes up after the
+  // request's postgres pool has been closed and every query it makes lands on a
+  // dead pool. `shutdown` below is that seam.
+  const liveSandboxFibers = new Set<Fiber.Fiber<ExecuteResult, E>>();
   // Outcomes of executions that already settled (resumed to completion, hit a
   // new pause, or died while paused). MCP clients retry `resume` when a
   // response gets lost in transit; without this cache the retry of an
@@ -613,6 +639,7 @@ export const createExecutionEngine = <E extends Cause.YieldableError = CodeExecu
     fiber = yield* Effect.forkDetach(
       codeExecutor.execute(code, invoker).pipe(Effect.withSpan("executor.code.exec")),
     );
+    liveSandboxFibers.add(fiber);
 
     // When the fiber settles on its own (sandbox timeout, failure) while
     // pauses are still outstanding, drop them: getPausedExecution must not
@@ -624,6 +651,9 @@ export const createExecutionEngine = <E extends Cause.YieldableError = CodeExecu
       Fiber.await(sandboxFiber).pipe(
         Effect.flatMap((exit) =>
           Effect.sync(() => {
+            // Settled on its own — it can no longer touch the host's DB handle,
+            // so it is not `shutdown`'s problem any more.
+            liveSandboxFibers.delete(sandboxFiber);
             const outcome = Exit.map(
               exit,
               (result): ExecutionResult => ({ status: "completed", result }),
@@ -727,10 +757,33 @@ export const createExecutionEngine = <E extends Cause.YieldableError = CodeExecu
     return result;
   });
 
+  /**
+   * End this engine's sandbox fibers, and WAIT for them to finish unwinding.
+   *
+   * A host calls this when the scope that owns the engine's DB handle is about
+   * to close. Interruption is awaited rather than fired and forgotten: the point
+   * is that no sandbox fiber is still able to issue a query by the time the
+   * host's connection finalizer runs, so returning early would reopen the very
+   * race this closes.
+   *
+   * Paused executions are dropped with the fibers — a pause whose fiber has been
+   * interrupted can never consume a response, so leaving the entry behind would
+   * only let a later `resume` hand back a pause that cannot progress.
+   */
+  const shutdown: Effect.Effect<void> = Effect.suspend(() => {
+    const fibers = Array.from(liveSandboxFibers);
+    liveSandboxFibers.clear();
+    for (const [id, paused] of pausedExecutions) {
+      if (fibers.includes(paused.fiber)) pausedExecutions.delete(id);
+    }
+    return Fiber.interruptAll(fibers);
+  });
+
   return {
     execute: runInlineExecution,
     executeWithPause: startPausableExecution,
     resume: resumeExecution,
+    shutdown,
     isExecutionSettled: (executionId) => Effect.sync(() => settledExecutionIds.has(executionId)),
     getPausedExecution: (executionId) =>
       Effect.sync(() => pausedExecutions.get(executionId) ?? null),

packages/core/execution/src/tool-invoker.test.ts

@@ -27,6 +27,7 @@ import {
   typeCheckOutputTypeScript,
 } from "@executor-js/sdk/testing";
 import { makeQuickJsExecutor } from "@executor-js/runtime-quickjs";
+import type { CodeExecutor, ExecuteResult } from "@executor-js/codemode-core";
 import { createExecutionEngine } from "./engine";
 import { ExecutionToolError } from "./errors";
 import {
@@ -1875,4 +1876,104 @@ describe("pause/resume with multiple elicitations", () => {
     expect(resumed.result.error).toBeUndefined();
     expect(resumed.result.result).toMatchObject({ ok: true });
   }, 10000);
+
+  // -------------------------------------------------------------------------
+  // Regression: a paused execution's sandbox fiber must not outlive the
+  // database handle it is holding.
+  //
+  // `executeWithPause` forks the sandbox with `Effect.forkDetach` so a pause can
+  // outlive the caller that observed it. What that fiber closes over is the
+  // problem: its invoker is built from the engine's `executor`, and the FumaDB
+  // client captures its handle at CONSTRUCTION rather than resolving it per
+  // operation. So the parked fiber keeps a live reference to whichever
+  // connection the host opened for the scope that built this engine.
+  //
+  // On the cloud `/api/*` plane that scope is the HTTP request, and its
+  // finalizer closes the postgres pool as soon as the response is written. A
+  // sandbox fiber still parked at that moment issues its next query against a
+  // pool that has already had `end()` called on it — which postgres.js answers
+  // with CONNECTION_ENDED, and which workerd answers with "Cannot perform I/O on
+  // behalf of a different request" once the socket is reached from a later
+  // request. Those are the storage faults seen in production on reads like
+  // `tool.findMany` and `plugin_storage.findMany`.
+  //
+  // `engine.shutdown` is the seam the host uses to end the fiber BEFORE closing
+  // the connection. Nothing is lost by ending it here: on a per-request engine
+  // the pause is already unreachable once the response is written (a resume
+  // lands on a different engine and replays the call instead).
+  // -------------------------------------------------------------------------
+  it.effect(
+    "shutdown ends a paused execution's sandbox fiber so it cannot outlive the request",
+    () =>
+      Effect.gen(function* () {
+        const executor = yield* makeElicitingExecutor();
+
+        // Instrumented in place of the QuickJS runtime so the test can observe
+        // the one thing that actually matters: whether the forked sandbox fiber
+        // is still alive. It pauses through the real approval-gated tool, so
+        // `executeWithPause` returns on the caller's own fiber exactly as it
+        // does in the HTTP handler.
+        let sandboxEnded = false;
+        const probeCodeExecutor: CodeExecutor<ExecutionToolError> = {
+          execute: (_code, toolInvoker) =>
+            Effect.gen(function* () {
+              yield* Effect.orDie(
+                toolInvoker.invoke({ path: "api.org.main.singleApproval", args: {} }),
+              );
+              return { result: "unreachable", logs: [] } satisfies ExecuteResult;
+            }).pipe(
+              Effect.onInterrupt(() =>
+                Effect.sync(() => {
+                  sandboxEnded = true;
+                }),
+              ),
+            ),
+        };
+
+        const engine = createExecutionEngine({ executor, codeExecutor: probeCodeExecutor });
+
+        const outcome = yield* engine.executeWithPause("ignored by the probe executor");
+        expect(outcome.status).toBe("paused");
+        const paused = outcome as Extract<typeof outcome, { status: "paused" }>;
+        expect(yield* engine.pausedExecutionCount()).toBe(1);
+
+        // This is the leak: the response is ready, but the sandbox is still
+        // parked on the approval, still holding the executor and its handle.
+        expect(sandboxEnded, "the sandbox is still alive while the request is served").toBe(false);
+
+        // Called from the same fiber that forked the sandbox — the shape the
+        // HTTP middleware runs `Effect.ensuring(engine.shutdown)` in. Bounded,
+        // because a shutdown that blocked would hold the request open for as
+        // long as the sandbox sat parked, which is worse than the leak.
+        const ended = yield* Effect.race(
+          engine.shutdown.pipe(Effect.as("ended" as const)),
+          Effect.sleep("5 seconds").pipe(Effect.as("hung" as const)),
+        );
+        expect(ended, "shutdown must not block the request it runs in").toBe("ended");
+
+        // The assertion the fix exists for: the fiber is actually gone by the
+        // time shutdown returns, so it can no longer reach the executor — and
+        // therefore can no longer query the connection the host closes next.
+        expect(sandboxEnded, "shutdown interrupted the parked sandbox fiber").toBe(true);
+
+        // The pause goes with it: one whose fiber is dead can never consume a
+        // response, so leaving it behind would only mislead a later resume.
+        expect(yield* engine.pausedExecutionCount()).toBe(0);
+        expect(yield* engine.getPausedExecution(paused.execution.id)).toBeNull();
+      }),
+    { timeout: 15000 },
+  );
+
+  it.effect("shutdown is a no-op when nothing is in flight", () =>
+    Effect.gen(function* () {
+      const executor = yield* makeElicitingExecutor();
+      const engine = createExecutionEngine({ executor, codeExecutor });
+
+      // Runs on every request, including the overwhelming majority that never
+      // pause, so it has to be inert rather than an error path.
+      yield* engine.shutdown;
+      yield* engine.shutdown;
+      expect(yield* engine.pausedExecutionCount()).toBe(0);
+    }),
+  );
 });

packages/hosts/cloudflare/src/mcp/agent-session-durable-object.test.ts

@@ -173,6 +173,8 @@ const makeEngine = (
       pausedExecutionCount: () => Effect.succeed(0),
       hasPausedExecutions: () => Effect.succeed(false),
       getDescription: Effect.succeed("test engine"),
+      // The fake forks nothing, so there is no sandbox fiber to end.
+      shutdown: Effect.void,
     },
   };
 };

packages/hosts/cloudflare/src/mcp/agent-session-model-resume.test.ts

@@ -263,6 +263,8 @@ const makeEngine = (
     pausedExecutionCount: () => Effect.succeed(0),
     hasPausedExecutions: () => Effect.succeed(false),
     getDescription: Effect.succeed("test engine"),
+    // The fake forks nothing, so there is no sandbox fiber to end.
+    shutdown: Effect.void,
   };
   return { calls, engine, resume };
 };

packages/hosts/mcp-apps-shell/src/shell-resource.smoke.test.ts

@@ -28,6 +28,8 @@ const stubEngine: ExecutionEngine<never> = {
   pausedExecutionCount: () => Effect.succeed(0),
   hasPausedExecutions: () => Effect.succeed(false),
   getDescription: Effect.succeed("smoke"),
+  // The fake forks nothing, so there is no sandbox fiber to end.
+  shutdown: Effect.void,
 };
 
 // oxlint-disable-next-line executor/no-double-cast -- boundary: MCP SDK ClientCapabilities predates the ext-apps `extensions` field

packages/hosts/mcp-apps-shell/src/shell/mcp-app.browser.test.ts

@@ -1311,6 +1311,8 @@ const startMcpHarness = async (openApi: OpenApiServer): Promise<McpHarness> => {
 const startSchemaElicitationMcpHarness = (): Promise<McpHarness> =>
   startMcpHarnessForEngine({
     getDescription: Effect.succeed("schema elicitation test executor"),
+    // The fake forks nothing, so there is no sandbox fiber to end.
+    shutdown: Effect.void,
     execute: () => Effect.succeed({ result: null }),
     executeWithPause: () =>
       Effect.succeed(

packages/hosts/mcp/src/artifacts-tools.test.ts

@@ -54,6 +54,8 @@ const makeStubEngine = <E extends Cause.YieldableError = never>(overrides: {
   pausedExecutionCount: () => Effect.succeed(0),
   hasPausedExecutions: () => Effect.succeed(false),
   getDescription: Effect.succeed("test executor"),
+  // The fake forks nothing, so there is no sandbox fiber to end.
+  shutdown: Effect.void,
 });
 
 /**

packages/hosts/mcp/src/namespace-search-tools.test.ts

@@ -35,6 +35,8 @@ const makeRecordingEngine = (): {
       pausedExecutionCount: () => Effect.succeed(0),
       hasPausedExecutions: () => Effect.succeed(false),
       getDescription: Effect.succeed("test executor"),
+      // The fake forks nothing, so there is no sandbox fiber to end.
+      shutdown: Effect.void,
     },
   };
 };

packages/hosts/mcp/src/tool-server.test.ts

@@ -56,6 +56,8 @@ const makeStubEngine = <E extends Cause.YieldableError = never>(overrides: {
   pausedExecutionCount: () => Effect.succeed(0),
   hasPausedExecutions: () => Effect.succeed(false),
   getDescription: Effect.succeed(overrides.description ?? "test executor"),
+  // The fake forks nothing, so there is no sandbox fiber to end.
+  shutdown: Effect.void,
 });
 
 type TestServerConfig<E extends Cause.YieldableError> = Pick<