From 7c510676a687ea91f8cec14e519613bbad3151af Mon Sep 17 00:00:00 2001 From: katereznykova Date: Tue, 3 Mar 2026 23:37:26 +0000 Subject: [PATCH 1/4] Tie transport retry budget to startup timeouts The transport layer's 503 retry budget was hard-coded at 120s, independent of the configurable container startup timeouts. Customers with large images who set longer startup timeouts still saw failures because the transport gave up before the container finished starting. The retry budget now scales to the sum of instanceGetTimeoutMS and portReadyTimeoutMS plus a 30s margin, with a 120s floor to preserve existing behavior at default settings. --- packages/sandbox/src/clients/base-client.ts | 3 ++- packages/sandbox/src/clients/sandbox-client.ts | 3 ++- .../src/clients/transport/base-transport.ts | 6 ++++-- .../sandbox/src/clients/transport/types.ts | 6 ++++++ packages/sandbox/src/clients/types.ts | 6 ++++++ packages/sandbox/src/sandbox.ts | 18 ++++++++++++++++++ 6 files changed, 38 insertions(+), 4 deletions(-) diff --git a/packages/sandbox/src/clients/base-client.ts b/packages/sandbox/src/clients/base-client.ts index dce838df5..c5cc0ce57 100644 --- a/packages/sandbox/src/clients/base-client.ts +++ b/packages/sandbox/src/clients/base-client.ts @@ -37,7 +37,8 @@ export abstract class BaseHttpClient { wsUrl: options.wsUrl, logger: this.logger, stub: options.stub, - port: options.port + port: options.port, + retryTimeoutMs: options.retryTimeoutMs }); } } diff --git a/packages/sandbox/src/clients/sandbox-client.ts b/packages/sandbox/src/clients/sandbox-client.ts index 6eed5796b..549729217 100644 --- a/packages/sandbox/src/clients/sandbox-client.ts +++ b/packages/sandbox/src/clients/sandbox-client.ts @@ -48,7 +48,8 @@ export class SandboxClient { baseUrl: options.baseUrl, logger: options.logger, stub: options.stub, - port: options.port + port: options.port, + retryTimeoutMs: options.retryTimeoutMs }); } diff --git a/packages/sandbox/src/clients/transport/base-transport.ts b/packages/sandbox/src/clients/transport/base-transport.ts index 68cde407e..5d299ea88 100644 --- a/packages/sandbox/src/clients/transport/base-transport.ts +++ b/packages/sandbox/src/clients/transport/base-transport.ts @@ -5,7 +5,7 @@ import type { ITransport, TransportConfig, TransportMode } from './types'; /** * Container startup retry configuration */ -const TIMEOUT_MS = 120_000; // 2 minutes total retry budget +const DEFAULT_RETRY_TIMEOUT_MS = 120_000; // 2 minutes total retry budget const MIN_TIME_FOR_RETRY_MS = 15_000; // Need at least 15s remaining to retry /** @@ -17,10 +17,12 @@ const MIN_TIME_FOR_RETRY_MS = 15_000; // Need at least 15s remaining to retry export abstract class BaseTransport implements ITransport { protected config: TransportConfig; protected logger: Logger; + private retryTimeoutMs: number; constructor(config: TransportConfig) { this.config = config; this.logger = config.logger ?? createNoOpLogger(); + this.retryTimeoutMs = config.retryTimeoutMs ?? DEFAULT_RETRY_TIMEOUT_MS; } abstract getMode(): TransportMode; @@ -44,7 +46,7 @@ export abstract class BaseTransport implements ITransport { // Check for retryable 503 (container starting) if (response.status === 503) { const elapsed = Date.now() - startTime; - const remaining = TIMEOUT_MS - elapsed; + const remaining = this.retryTimeoutMs - elapsed; if (remaining > MIN_TIME_FOR_RETRY_MS) { const delay = Math.min(3000 * 2 ** attempt, 30000); diff --git a/packages/sandbox/src/clients/transport/types.ts b/packages/sandbox/src/clients/transport/types.ts index 7eb57eb75..f1f5a313a 100644 --- a/packages/sandbox/src/clients/transport/types.ts +++ b/packages/sandbox/src/clients/transport/types.ts @@ -30,6 +30,12 @@ export interface TransportConfig { /** Connection timeout in milliseconds (WebSocket only) */ connectTimeoutMs?: number; + + /** Total retry budget in milliseconds for 503 retries during container startup. + * Defaults to 120_000 (2 minutes). Should be at least as large as the sum of + * instanceGetTimeoutMS + portReadyTimeoutMS to avoid the client giving up + * before the container has finished starting. */ + retryTimeoutMs?: number; } /** diff --git a/packages/sandbox/src/clients/types.ts b/packages/sandbox/src/clients/types.ts index d3d098b3c..c3fdcc40c 100644 --- a/packages/sandbox/src/clients/types.ts +++ b/packages/sandbox/src/clients/types.ts @@ -53,6 +53,12 @@ export interface HttpClientOptions { * When provided, clients will use this transport instead of creating their own. */ transport?: ITransport; + + /** + * Total retry budget in milliseconds for 503 retries during container startup. + * Passed through to the transport layer. Defaults to 120_000 (2 minutes). + */ + retryTimeoutMs?: number; } /** diff --git a/packages/sandbox/src/sandbox.ts b/packages/sandbox/src/sandbox.ts index fad845382..40a0714db 100644 --- a/packages/sandbox/src/sandbox.ts +++ b/packages/sandbox/src/sandbox.ts @@ -320,10 +320,21 @@ export class Sandbox extends Container implements ISandbox { * Create a SandboxClient with current transport settings */ private createSandboxClient(): SandboxClient { + // Retry budget must cover the full container startup window (instance + // provisioning + port readiness) plus a margin for backoff delays, + // so the client doesn't give up before the DO finishes starting + // the container. 30 seconds of margin covers the maximum single + // backoff delay (capped at 30s in BaseTransport). + const startupBudgetMs = + this.containerTimeouts.instanceGetTimeoutMS + + this.containerTimeouts.portReadyTimeoutMS; + const retryTimeoutMs = Math.max(120_000, startupBudgetMs + 30_000); + return new SandboxClient({ logger: this.logger, port: 3000, stub: this, + retryTimeoutMs, ...(this.transport === 'websocket' && { transportMode: 'websocket' as const, wsUrl: 'ws://localhost:3000/ws' @@ -408,6 +419,9 @@ export class Sandbox extends Container implements ISandbox { ...this.containerTimeouts, ...storedTimeouts }; + // Recreate client so the transport retry budget reflects stored timeouts + this.client = this.createSandboxClient(); + this.codeInterpreter = new CodeInterpreter(this); } }); } @@ -530,6 +544,10 @@ export class Sandbox extends Container implements ISandbox { // Persist to storage await this.ctx.storage.put('containerTimeouts', this.containerTimeouts); + // Recreate client so the transport retry budget reflects new timeouts + this.client = this.createSandboxClient(); + this.codeInterpreter = new CodeInterpreter(this); + this.logger.debug('Container timeouts updated', this.containerTimeouts); } From 50b2951a965dc9295fea5ad3a1ca8222a62ac53e Mon Sep 17 00:00:00 2001 From: katereznykova Date: Tue, 3 Mar 2026 23:41:51 +0000 Subject: [PATCH 2/4] Add changeset --- .changeset/fix-transport-retry-budget.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/fix-transport-retry-budget.md diff --git a/.changeset/fix-transport-retry-budget.md b/.changeset/fix-transport-retry-budget.md new file mode 100644 index 000000000..0b44fd4a9 --- /dev/null +++ b/.changeset/fix-transport-retry-budget.md @@ -0,0 +1,5 @@ +--- +'@cloudflare/sandbox': patch +--- + +Fix sandbox creation timing out for large container images even when startup timeouts are configured to allow enough time. The transport retry budget now automatically scales to match configured startup timeouts instead of being hard-coded at 120 seconds. From b6ef102b6a5f61ff0c558ac646d7068dc04e195b Mon Sep 17 00:00:00 2001 From: "ask-bonk[bot]" Date: Thu, 5 Mar 2026 11:38:26 +0000 Subject: [PATCH 3/4] Add retry timeout setter to avoid client recreation The transport retry budget can now be updated in place via setRetryTimeoutMs(), so timeout changes no longer require recreating the entire client and interpreter graph. This removes the risk of stale references when components snapshot the client at construction time. --- packages/sandbox/src/clients/base-client.ts | 7 ++++ .../sandbox/src/clients/sandbox-client.ts | 26 +++++++++++++++ .../src/clients/transport/base-transport.ts | 4 +++ .../sandbox/src/clients/transport/types.ts | 5 +++ packages/sandbox/src/sandbox.ts | 33 ++++++++++--------- 5 files changed, 60 insertions(+), 15 deletions(-) diff --git a/packages/sandbox/src/clients/base-client.ts b/packages/sandbox/src/clients/base-client.ts index c5cc0ce57..f3122b7eb 100644 --- a/packages/sandbox/src/clients/base-client.ts +++ b/packages/sandbox/src/clients/base-client.ts @@ -43,6 +43,13 @@ export abstract class BaseHttpClient { } } + /** + * Update the transport's 503 retry budget + */ + setRetryTimeoutMs(ms: number): void { + this.transport.setRetryTimeoutMs(ms); + } + /** * Check if using WebSocket transport */ diff --git a/packages/sandbox/src/clients/sandbox-client.ts b/packages/sandbox/src/clients/sandbox-client.ts index 549729217..18c9cf211 100644 --- a/packages/sandbox/src/clients/sandbox-client.ts +++ b/packages/sandbox/src/clients/sandbox-client.ts @@ -74,6 +74,32 @@ export class SandboxClient { this.watch = new WatchClient(clientOptions); } + /** + * Update the 503 retry budget on all transports without recreating the client. + * + * In WebSocket mode a single shared transport is used, so one update covers + * every sub-client. In HTTP mode each sub-client owns its own transport, so + * all of them are updated individually. + */ + setRetryTimeoutMs(ms: number): void { + if (this.transport) { + // WebSocket mode — single shared transport + this.transport.setRetryTimeoutMs(ms); + } else { + // HTTP mode — each sub-client has its own transport + this.backup.setRetryTimeoutMs(ms); + this.commands.setRetryTimeoutMs(ms); + this.files.setRetryTimeoutMs(ms); + this.processes.setRetryTimeoutMs(ms); + this.ports.setRetryTimeoutMs(ms); + this.git.setRetryTimeoutMs(ms); + this.interpreter.setRetryTimeoutMs(ms); + this.utils.setRetryTimeoutMs(ms); + this.desktop.setRetryTimeoutMs(ms); + this.watch.setRetryTimeoutMs(ms); + } + } + /** * Get the current transport mode */ diff --git a/packages/sandbox/src/clients/transport/base-transport.ts b/packages/sandbox/src/clients/transport/base-transport.ts index 5d299ea88..fe3a9996a 100644 --- a/packages/sandbox/src/clients/transport/base-transport.ts +++ b/packages/sandbox/src/clients/transport/base-transport.ts @@ -30,6 +30,10 @@ export abstract class BaseTransport implements ITransport { abstract disconnect(): void; abstract isConnected(): boolean; + setRetryTimeoutMs(ms: number): void { + this.retryTimeoutMs = ms; + } + /** * Fetch with automatic retry for 503 (container starting) * diff --git a/packages/sandbox/src/clients/transport/types.ts b/packages/sandbox/src/clients/transport/types.ts index f1f5a313a..5c6beaeb1 100644 --- a/packages/sandbox/src/clients/transport/types.ts +++ b/packages/sandbox/src/clients/transport/types.ts @@ -80,4 +80,9 @@ export interface ITransport { * Check if connected (always true for HTTP) */ isConnected(): boolean; + + /** + * Update the 503 retry budget without recreating the transport + */ + setRetryTimeoutMs(ms: number): void; } diff --git a/packages/sandbox/src/sandbox.ts b/packages/sandbox/src/sandbox.ts index 40a0714db..73ec27d7f 100644 --- a/packages/sandbox/src/sandbox.ts +++ b/packages/sandbox/src/sandbox.ts @@ -317,24 +317,29 @@ export class Sandbox extends Container implements ISandbox { } /** - * Create a SandboxClient with current transport settings + * Compute the transport retry budget from current container timeouts. + * + * The budget covers the full container startup window (instance provisioning + * + port readiness) plus a 30s margin for the maximum single backoff delay + * (capped at 30s in BaseTransport). The 120s floor preserves the previous + * default for short timeout configurations. */ - private createSandboxClient(): SandboxClient { - // Retry budget must cover the full container startup window (instance - // provisioning + port readiness) plus a margin for backoff delays, - // so the client doesn't give up before the DO finishes starting - // the container. 30 seconds of margin covers the maximum single - // backoff delay (capped at 30s in BaseTransport). + private computeRetryTimeoutMs(): number { const startupBudgetMs = this.containerTimeouts.instanceGetTimeoutMS + this.containerTimeouts.portReadyTimeoutMS; - const retryTimeoutMs = Math.max(120_000, startupBudgetMs + 30_000); + return Math.max(120_000, startupBudgetMs + 30_000); + } + /** + * Create a SandboxClient with current transport settings + */ + private createSandboxClient(): SandboxClient { return new SandboxClient({ logger: this.logger, port: 3000, stub: this, - retryTimeoutMs, + retryTimeoutMs: this.computeRetryTimeoutMs(), ...(this.transport === 'websocket' && { transportMode: 'websocket' as const, wsUrl: 'ws://localhost:3000/ws' @@ -419,9 +424,8 @@ export class Sandbox extends Container implements ISandbox { ...this.containerTimeouts, ...storedTimeouts }; - // Recreate client so the transport retry budget reflects stored timeouts - this.client = this.createSandboxClient(); - this.codeInterpreter = new CodeInterpreter(this); + // Update the transport retry budget to reflect stored timeouts + this.client.setRetryTimeoutMs(this.computeRetryTimeoutMs()); } }); } @@ -544,9 +548,8 @@ export class Sandbox extends Container implements ISandbox { // Persist to storage await this.ctx.storage.put('containerTimeouts', this.containerTimeouts); - // Recreate client so the transport retry budget reflects new timeouts - this.client = this.createSandboxClient(); - this.codeInterpreter = new CodeInterpreter(this); + // Update the transport retry budget to reflect new timeouts + this.client.setRetryTimeoutMs(this.computeRetryTimeoutMs()); this.logger.debug('Container timeouts updated', this.containerTimeouts); } From 285a70837ec5cd4b9c79f3049146ed3715d91552 Mon Sep 17 00:00:00 2001 From: "ask-bonk[bot]" Date: Thu, 5 Mar 2026 11:38:50 +0000 Subject: [PATCH 4/4] Add retry timeout setter; stop recreating client. Co-authored-by: ghostwriternr --- package-lock.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/package-lock.json b/package-lock.json index bfb99c92f..2ad88a1dd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10856,6 +10856,7 @@ "os": [ "android" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -10876,6 +10877,7 @@ "os": [ "darwin" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -10896,6 +10898,7 @@ "os": [ "darwin" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -10916,6 +10919,7 @@ "os": [ "freebsd" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -10936,6 +10940,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -10956,6 +10961,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -10976,6 +10982,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -10996,6 +11003,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -11016,6 +11024,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -11036,6 +11045,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -11056,6 +11066,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">= 12.0.0" },