fix(selfhost): refresh stale connection tools concurrently and expose EXECUTOR_TOOLS_SYNC_TTL_MS

.changeset/concurrent-stale-tools-sync.md

@@ -0,0 +1,11 @@
+---
+"executor": patch
+---
+
+**Stale tool catalogs refresh together instead of one after another, and self-host can set the freshness window**
+
+A tools read rebuilds every connection whose catalog has gone stale. Those rebuilds each dial their own upstream, but ran strictly one after another, so a host with several stale remote catalogs paid the sum of every server's latency on the read that tripped the TTL. The upstream listings now run concurrently, bounded so a large stale set cannot open an unbounded number of listings from one read.
+
+Only the listings overlap. Each rebuild's catalog write stays single-file, because a self-host database is one connection issuing raw `BEGIN`/`COMMIT` and a second transaction opened while one is live fails outright. A rebuild that fails now also logs a warning naming the connection and the reason, instead of disappearing: the read still succeeds on the stale-but-working catalog and the other connections still finish, but a permanently broken connection no longer re-fails silently on every read.
+
+Self-host also exposes the freshness window as `EXECUTOR_TOOLS_SYNC_TTL_MS`. Leave it unset for the 15-minute default, or set `off` (equivalently `null` or `false`, in any case) to disable time-based re-sync and leave stale-marking and config revision as the only refresh triggers. The value forwards to the SDK verbatim, so `0` keeps its SDK meaning: every catalog is expired on every read. A malformed, negative, or too-large-to-represent value is refused at boot rather than silently falling back to the default.

apps/host-selfhost/src/config.ts

@@ -51,6 +51,12 @@ export interface SelfHostConfig {
    * minutes (the same pattern as MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS on cloud).
    */
   readonly sandboxTimeoutMs: number | undefined;
+  /**
+   * How long a connection's persisted remote tool catalog stays fresh, in ms.
+   * `undefined` takes the SDK default (15 minutes); `null` disables time-based
+   * re-sync, leaving stale-marking and config revision as the only triggers.
+   */
+  readonly toolsSyncTtlMs: number | null | undefined;
 }
 
 export const resolveDataDir = (): string =>
@@ -157,6 +163,7 @@ export const loadConfig = (): SelfHostConfig => {
     organizationName: process.env.EXECUTOR_ORG_NAME ?? "Default",
     orgSlug: resolveOrgSlug(),
     sandboxTimeoutMs: resolveSandboxTimeoutMs(),
+    toolsSyncTtlMs: resolveToolsSyncTtlMs(),
   };
 };
 
@@ -190,3 +197,42 @@ const resolveOrgSlug = (): string => {
   }
   return slug;
 };
+
+// EXECUTOR_TOOLS_SYNC_TTL_MS — how long a remote tool catalog (an MCP server's
+// tool set, which changes server-side with no executor-visible signal) stays
+// fresh before the next tools read re-lists it. Unset takes the SDK default of
+// 15 minutes.
+//
+// The value forwards to the SDK's `toolsSyncTtlMs` verbatim, so `0` keeps the
+// SDK's meaning — every catalog is expired on every read. "off", "null" and
+// "false" disable time-based re-sync (the SDK's `null` sentinel), since
+// operators reach for all three spellings. The comparison is case-insensitive:
+// "OFF" and "False" are the same intent typed by a different operator.
+//
+// Like the other knobs here a malformed or negative value is refused rather
+// than silently ignored: an operator who sets the TTL and typos it should find
+// out at boot, not by wondering months later why catalogs never refresh.
+const TOOLS_SYNC_TTL_DISABLE_TOKENS = new Set(["off", "null", "false"]);
+
+const resolveToolsSyncTtlMs = (): number | null | undefined => {
+  const raw = process.env.EXECUTOR_TOOLS_SYNC_TTL_MS?.trim();
+  if (!raw) return undefined;
+  if (TOOLS_SYNC_TTL_DISABLE_TOKENS.has(raw.toLowerCase())) return null;
+  const parsed = Number(raw);
+  // `isSafeInteger`, not `isInteger`: past 2^53 a decimal literal silently
+  // rounds to a nearby representable value, so an operator's typo'd digit
+  // would boot as a TTL they never wrote. Refuse it instead.
+  if (!Number.isSafeInteger(parsed)) {
+    // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: refuse to boot on a malformed operator knob
+    throw new Error(
+      `EXECUTOR_TOOLS_SYNC_TTL_MS ${JSON.stringify(raw)} is not an exactly representable whole number of milliseconds ("off", "null" or "false" disable time-based re-sync)`,
+    );
+  }
+  if (parsed < 0) {
+    // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: refuse to boot on a malformed operator knob
+    throw new Error(
+      `EXECUTOR_TOOLS_SYNC_TTL_MS ${JSON.stringify(raw)} must not be negative (use "off" to disable time-based re-sync)`,
+    );
+  }
+  return parsed;
+};

apps/host-selfhost/src/execution.ts

@@ -55,6 +55,7 @@ export const SelfHostHostConfig: Layer.Layer<HostConfig> = Layer.sync(HostConfig
     allowLocalNetwork: config.allowLocalNetwork,
     webBaseUrl: config.webBaseUrl,
     oauthCallbackPath: "/api/oauth/callback",
+    toolsSyncTtlMs: config.toolsSyncTtlMs,
     onIntegrationChange: (event) =>
       selfHostAnalytics.record(
         event.kind === "added" ? "integration_added" : "integration_removed",

apps/host-selfhost/src/executor-config.test.ts

@@ -1,11 +1,14 @@
 import { afterEach, beforeEach, expect, test } from "@effect/vitest";
 
+import { loadConfig } from "./config";
 import executorConfig from "../executor.config";
 
 const ENV_NAME = "EXECUTOR_ALLOW_STDIO_MCP";
 const SECRET_ENV_NAME = "EXECUTOR_SECRET_KEY";
+const TTL_ENV_NAME = "EXECUTOR_TOOLS_SYNC_TTL_MS";
 const originalValue = process.env[ENV_NAME];
 const originalSecret = process.env[SECRET_ENV_NAME];
+const originalTtl = process.env[TTL_ENV_NAME];
 
 beforeEach(() => {
   process.env[SECRET_ENV_NAME] = originalSecret ?? "executor-config-test-secret";
@@ -22,6 +25,11 @@ afterEach(() => {
   } else {
     process.env[SECRET_ENV_NAME] = originalSecret;
   }
+  if (originalTtl === undefined) {
+    delete process.env[TTL_ENV_NAME];
+  } else {
+    process.env[TTL_ENV_NAME] = originalTtl;
+  }
 });
 
 const allowStdio = (): boolean => {
@@ -57,3 +65,50 @@ test("stdio MCP is enabled when the opt-in is exactly true", () => {
   process.env[ENV_NAME] = "true";
   expect(allowStdio()).toBe(true);
 });
+
+test("an unset tools-sync TTL leaves the SDK default in place", () => {
+  delete process.env[TTL_ENV_NAME];
+  expect(loadConfig().toolsSyncTtlMs).toBeUndefined();
+
+  process.env[TTL_ENV_NAME] = "   ";
+  expect(loadConfig().toolsSyncTtlMs).toBeUndefined();
+});
+
+test("a positive tools-sync TTL is forwarded verbatim", () => {
+  process.env[TTL_ENV_NAME] = "60000";
+  expect(loadConfig().toolsSyncTtlMs).toBe(60000);
+});
+
+// 0 keeps the SDK's own meaning — every catalog is expired on every read —
+// so the env var never means the opposite of the config field it feeds.
+test("a zero tools-sync TTL forwards as the SDK's always-stale 0", () => {
+  process.env[TTL_ENV_NAME] = "0";
+  expect(loadConfig().toolsSyncTtlMs).toBe(0);
+});
+
+// Case-insensitive: the disable tokens are operator intent, not a keyword, and
+// "OFF" typed in a systemd unit means what "off" means in a .env file.
+test.each(["off", "null", "false", "OFF", "Null", "FALSE", "  Off  "])(
+  "the tools-sync TTL is disabled by %s",
+  (raw) => {
+    process.env[TTL_ENV_NAME] = raw;
+    expect(loadConfig().toolsSyncTtlMs).toBeNull();
+  },
+);
+
+// A typo'd knob must not silently degrade into the 15-minute default; the
+// operator finds out at boot instead of wondering why catalogs never refresh.
+// "9007199254740993" and "1e30" are whole numbers that no longer round-trip
+// through a double — accepting them would boot a TTL the operator never wrote.
+test.each(["abc", "60_000", "1.5", "1e3ms", "NaN", "Infinity", "9007199254740993", "1e30"])(
+  "a malformed tools-sync TTL (%s) refuses to boot",
+  (raw) => {
+    process.env[TTL_ENV_NAME] = raw;
+    expect(() => loadConfig()).toThrow(/EXECUTOR_TOOLS_SYNC_TTL_MS/);
+  },
+);
+
+test("a negative tools-sync TTL refuses to boot", () => {
+  process.env[TTL_ENV_NAME] = "-1";
+  expect(() => loadConfig()).toThrow(/must not be negative/);
+});

packages/core/api/src/server/scoped-executor.ts

@@ -115,6 +115,14 @@ export interface HostConfigShape {
    * attempted as it was before the gate existed.
    */
   readonly enterpriseManagedRollout?: ExecutorConfig["enterpriseManagedRollout"];
+  /**
+   * Forwarded verbatim to `ExecutorConfig.toolsSyncTtlMs`: how long a
+   * connection's persisted remote tool catalog stays fresh. Omit to take the
+   * SDK default (15 minutes); `null` disables time-based re-sync. Declared
+   * here — not per-request — because catalog freshness is a deployment-wide
+   * operator knob.
+   */
+  readonly toolsSyncTtlMs?: number | null;
 }
 
 export class HostConfig extends Context.Service<HostConfig, HostConfigShape>()(
@@ -296,6 +304,7 @@ export const makeScopedExecutor = <
       httpClientLayer,
       fetch: hostedFetch,
       onIntegrationChange: config.onIntegrationChange,
+      ...(config.toolsSyncTtlMs !== undefined ? { toolsSyncTtlMs: config.toolsSyncTtlMs } : {}),
       onElicitation: "accept-all",
       redirectUri,
       oauthCallbackStateOrgSlug: orgSlug,