From 1fb6b4c490c48308b932c143290b84bbd7323014 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Fri, 28 Aug 2026 05:46:03 -0700 Subject: [PATCH 1/3] Add tests for bounded hub shutdown drain Covers drainWithTimeout's drained/timed-out/failed outcomes and shutdownHub's exit-code and reportError wiring for each, with injected exit/report callbacks and short timeouts so the suite runs in milliseconds. --- apps/hub/src/shutdown.test.ts | 78 +++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 apps/hub/src/shutdown.test.ts diff --git a/apps/hub/src/shutdown.test.ts b/apps/hub/src/shutdown.test.ts new file mode 100644 index 000000000..fb70a27c1 --- /dev/null +++ b/apps/hub/src/shutdown.test.ts @@ -0,0 +1,78 @@ +import { expect, test } from "bun:test"; +import { drainWithTimeout, shutdownHub } from "./shutdown"; + +test("drainWithTimeout resolves drained when the drain completes inside the bound", async () => { + const outcome = await drainWithTimeout(() => Promise.resolve(), 1_000); + expect(outcome).toEqual({ kind: "drained" }); +}); + +test("drainWithTimeout resolves timed-out when the drain outlives the bound", async () => { + const outcome = await drainWithTimeout( + () => new Promise(() => undefined), + 10, + ); + expect(outcome).toEqual({ kind: "timed-out" }); +}); + +test("drainWithTimeout resolves failed with the thrown error when the drain throws", async () => { + const error = new Error("drain fault"); + const outcome = await drainWithTimeout(() => Promise.reject(error), 1_000); + expect(outcome).toEqual({ kind: "failed", error }); +}); + +test("shutdownHub exits 0 when the drain resolves inside the bound", async () => { + let exitCode: number | undefined; + const reported: unknown[] = []; + await shutdownHub({ + drain: () => Promise.resolve(), + timeoutMs: 1_000, + exit: (code) => { + exitCode = code; + }, + report: (error) => { + reported.push(error); + return "unused-ref-id"; + }, + }); + expect(exitCode).toBe(0); + expect(reported).toEqual([]); +}); + +test("shutdownHub exits non-zero and reports the cause when the drain never settles", async () => { + let exitCode: number | undefined; + const reported: unknown[] = []; + await shutdownHub({ + drain: () => new Promise(() => undefined), + timeoutMs: 10, + exit: (code) => { + exitCode = code; + }, + report: (error) => { + reported.push(error); + return "unused-ref-id"; + }, + }); + expect(exitCode).toBe(1); + expect(reported).toHaveLength(1); + expect(reported[0]).toBeInstanceOf(Error); + expect((reported[0] as Error).message).toContain("exceeded 10ms"); +}); + +test("shutdownHub exits non-zero and reports the cause when the drain rejects", async () => { + let exitCode: number | undefined; + const reported: unknown[] = []; + const error = new Error("close fault"); + await shutdownHub({ + drain: () => Promise.reject(error), + timeoutMs: 1_000, + exit: (code) => { + exitCode = code; + }, + report: (reportedError) => { + reported.push(reportedError); + return "unused-ref-id"; + }, + }); + expect(exitCode).toBe(1); + expect(reported).toEqual([error]); +}); From 6ddc79d4aa97370953e87b538dc474d3e22d4da7 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Fri, 28 Aug 2026 05:46:09 -0700 Subject: [PATCH 2/3] Hub shutdown: bound the drain and always exit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hub shutdown previously awaited hub.close()'s eight sequential closes with no timeout: a hung close blocked SIGTERM exit forever, and a rejection was unhandled. shutdownHub now bounds the drain to 10s and always exits — 0 on a clean drain, non-zero with the cause reported through reportError on a throw or timeout — mirroring the sidecar's drainWithTimeout shape as a local copy rather than a shared import, since the two apps diverge on timeout exit semantics. Fixes CL-7133. --- apps/hub/src/index.ts | 9 +++-- apps/hub/src/shutdown.ts | 71 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 apps/hub/src/shutdown.ts diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 66db0cc6d..bd37716ef 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -354,6 +354,7 @@ import { createBootAssetWiring, REGISTRIES } from "./asset-service-factory"; import { createRoutineScheduler } from "./routine-scheduler"; import { createToolGrantsForPins } from "./tool-grants"; import { createMcpCredentialBindingsFor } from "./mcp-credential-bindings"; +import { shutdownHub } from "./shutdown"; // Host policy constants, not configuration. const MAX_TARBALL_BYTES = 10 * 1024 * 1024; @@ -3454,10 +3455,14 @@ if (import.meta.main) { }); const log = getLogger(["hub"]); log.info`Hub serving on port ${port}`; + const SHUTDOWN_DRAIN_MS = 10_000; const shutdown = async () => { await server.stop(); - await hub.close(); - process.exit(0); + await shutdownHub({ + drain: () => hub.close(), + timeoutMs: SHUTDOWN_DRAIN_MS, + exit: (code) => process.exit(code), + }); }; process.on("SIGINT", () => void shutdown()); process.on("SIGTERM", () => void shutdown()); diff --git a/apps/hub/src/shutdown.ts b/apps/hub/src/shutdown.ts new file mode 100644 index 000000000..b97be00fa --- /dev/null +++ b/apps/hub/src/shutdown.ts @@ -0,0 +1,71 @@ +// Bounded drain for process shutdown, mirroring apps/sidecar/src/shutdown.ts's +// shape. Kept as a local copy rather than a shared import or package: it's a +// ~15-line primitive, and the two apps already diverge on exit semantics +// (the sidecar treats a timeout as a clean-enough exit; the hub treats a +// timeout as a fault worth reporting), so sharing it would need a parameter +// immediately. +// +// The platform sends SIGTERM and expects a prompt exit; a drain that hangs +// would turn every deploy into an apparent crash, so the bound cuts it off +// and reports the cause instead of leaving an unhandled rejection. + +import { reportError } from "@corbits/error-sink"; + +export type DrainOutcome = + | { kind: "drained" } + | { kind: "timed-out" } + | { kind: "failed"; error: unknown }; + +export async function drainWithTimeout( + drain: () => Promise, + timeoutMs: number, +): Promise { + let timer: ReturnType | undefined; + const timedOut = new Promise((resolve) => { + timer = setTimeout(() => { + resolve({ kind: "timed-out" }); + }, timeoutMs); + }); + const drained = (async (): Promise => { + try { + await drain(); + return { kind: "drained" }; + } catch (error) { + return { kind: "failed", error }; + } + })(); + const outcome = await Promise.race([drained, timedOut]); + clearTimeout(timer); + return outcome; +} + +export type ShutdownHubDeps = { + drain: () => Promise; + timeoutMs: number; + exit: (code: number) => void; + report?: typeof reportError; +}; + +/** + * Drains the hub within `timeoutMs` and always exits: 0 on a clean drain, + * non-zero with the cause reported through `reportError` on a throw or a + * timeout. + */ +export async function shutdownHub({ + drain, + timeoutMs, + exit, + report = reportError, +}: ShutdownHubDeps): Promise { + const outcome = await drainWithTimeout(drain, timeoutMs); + if (outcome.kind === "drained") { + exit(0); + return; + } + const error = + outcome.kind === "timed-out" + ? new Error(`Hub shutdown drain exceeded ${timeoutMs}ms`) + : outcome.error; + report(error, { operation: "hub.shutdown" }); + exit(1); +} From 5c55ae12cef3b5fddf79d4b95451ebbd68a335ba Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Fri, 28 Aug 2026 08:27:07 -0700 Subject: [PATCH 3/3] Hub shutdown: bound server.stop inside the same drain Bun's Server.stop waits for open connections and websockets by default, so leaving it outside the bounded drain kept the hang CL-7133 was filed against, one line earlier. --- apps/hub/src/index.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index bd37716ef..5bf15edfd 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -3456,14 +3456,17 @@ if (import.meta.main) { const log = getLogger(["hub"]); log.info`Hub serving on port ${port}`; const SHUTDOWN_DRAIN_MS = 10_000; - const shutdown = async () => { - await server.stop(); - await shutdownHub({ - drain: () => hub.close(), + // `server.stop()` waits for open connections and websockets by default, + // so it sits inside the same bound as the hub's own closes. + const shutdown = () => + shutdownHub({ + drain: async () => { + await server.stop(); + await hub.close(); + }, timeoutMs: SHUTDOWN_DRAIN_MS, exit: (code) => process.exit(code), }); - }; process.on("SIGINT", () => void shutdown()); process.on("SIGTERM", () => void shutdown()); }