diff --git a/.changeset/fix-transport-retry-budget.md b/.changeset/fix-transport-retry-budget.md new file mode 100644 index 000000000..0b44fd4a9 --- /dev/null +++ b/.changeset/fix-transport-retry-budget.md @@ -0,0 +1,5 @@ +--- +'@cloudflare/sandbox': patch +--- + +Fix sandbox creation timing out for large container images even when startup timeouts are configured to allow enough time. The transport retry budget now automatically scales to match configured startup timeouts instead of being hard-coded at 120 seconds. diff --git a/package-lock.json b/package-lock.json index bfb99c92f..2ad88a1dd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10856,6 +10856,7 @@ "os": [ "android" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -10876,6 +10877,7 @@ "os": [ "darwin" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -10896,6 +10898,7 @@ "os": [ "darwin" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -10916,6 +10919,7 @@ "os": [ "freebsd" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -10936,6 +10940,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -10956,6 +10961,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -10976,6 +10982,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -10996,6 +11003,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -11016,6 +11024,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -11036,6 +11045,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -11056,6 +11066,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, diff --git a/packages/sandbox/src/clients/base-client.ts b/packages/sandbox/src/clients/base-client.ts index dce838df5..f3122b7eb 100644 --- a/packages/sandbox/src/clients/base-client.ts +++ b/packages/sandbox/src/clients/base-client.ts @@ -37,11 +37,19 @@ export abstract class BaseHttpClient { wsUrl: options.wsUrl, logger: this.logger, stub: options.stub, - port: options.port + port: options.port, + retryTimeoutMs: options.retryTimeoutMs }); } } + /** + * Update the transport's 503 retry budget + */ + setRetryTimeoutMs(ms: number): void { + this.transport.setRetryTimeoutMs(ms); + } + /** * Check if using WebSocket transport */ diff --git a/packages/sandbox/src/clients/sandbox-client.ts b/packages/sandbox/src/clients/sandbox-client.ts index 6eed5796b..18c9cf211 100644 --- a/packages/sandbox/src/clients/sandbox-client.ts +++ b/packages/sandbox/src/clients/sandbox-client.ts @@ -48,7 +48,8 @@ export class SandboxClient { baseUrl: options.baseUrl, logger: options.logger, stub: options.stub, - port: options.port + port: options.port, + retryTimeoutMs: options.retryTimeoutMs }); } @@ -73,6 +74,32 @@ export class SandboxClient { this.watch = new WatchClient(clientOptions); } + /** + * Update the 503 retry budget on all transports without recreating the client. + * + * In WebSocket mode a single shared transport is used, so one update covers + * every sub-client. In HTTP mode each sub-client owns its own transport, so + * all of them are updated individually. + */ + setRetryTimeoutMs(ms: number): void { + if (this.transport) { + // WebSocket mode — single shared transport + this.transport.setRetryTimeoutMs(ms); + } else { + // HTTP mode — each sub-client has its own transport + this.backup.setRetryTimeoutMs(ms); + this.commands.setRetryTimeoutMs(ms); + this.files.setRetryTimeoutMs(ms); + this.processes.setRetryTimeoutMs(ms); + this.ports.setRetryTimeoutMs(ms); + this.git.setRetryTimeoutMs(ms); + this.interpreter.setRetryTimeoutMs(ms); + this.utils.setRetryTimeoutMs(ms); + this.desktop.setRetryTimeoutMs(ms); + this.watch.setRetryTimeoutMs(ms); + } + } + /** * Get the current transport mode */ diff --git a/packages/sandbox/src/clients/transport/base-transport.ts b/packages/sandbox/src/clients/transport/base-transport.ts index 68cde407e..fe3a9996a 100644 --- a/packages/sandbox/src/clients/transport/base-transport.ts +++ b/packages/sandbox/src/clients/transport/base-transport.ts @@ -5,7 +5,7 @@ import type { ITransport, TransportConfig, TransportMode } from './types'; /** * Container startup retry configuration */ -const TIMEOUT_MS = 120_000; // 2 minutes total retry budget +const DEFAULT_RETRY_TIMEOUT_MS = 120_000; // 2 minutes total retry budget const MIN_TIME_FOR_RETRY_MS = 15_000; // Need at least 15s remaining to retry /** @@ -17,10 +17,12 @@ const MIN_TIME_FOR_RETRY_MS = 15_000; // Need at least 15s remaining to retry export abstract class BaseTransport implements ITransport { protected config: TransportConfig; protected logger: Logger; + private retryTimeoutMs: number; constructor(config: TransportConfig) { this.config = config; this.logger = config.logger ?? createNoOpLogger(); + this.retryTimeoutMs = config.retryTimeoutMs ?? DEFAULT_RETRY_TIMEOUT_MS; } abstract getMode(): TransportMode; @@ -28,6 +30,10 @@ export abstract class BaseTransport implements ITransport { abstract disconnect(): void; abstract isConnected(): boolean; + setRetryTimeoutMs(ms: number): void { + this.retryTimeoutMs = ms; + } + /** * Fetch with automatic retry for 503 (container starting) * @@ -44,7 +50,7 @@ export abstract class BaseTransport implements ITransport { // Check for retryable 503 (container starting) if (response.status === 503) { const elapsed = Date.now() - startTime; - const remaining = TIMEOUT_MS - elapsed; + const remaining = this.retryTimeoutMs - elapsed; if (remaining > MIN_TIME_FOR_RETRY_MS) { const delay = Math.min(3000 * 2 ** attempt, 30000); diff --git a/packages/sandbox/src/clients/transport/types.ts b/packages/sandbox/src/clients/transport/types.ts index 7eb57eb75..5c6beaeb1 100644 --- a/packages/sandbox/src/clients/transport/types.ts +++ b/packages/sandbox/src/clients/transport/types.ts @@ -30,6 +30,12 @@ export interface TransportConfig { /** Connection timeout in milliseconds (WebSocket only) */ connectTimeoutMs?: number; + + /** Total retry budget in milliseconds for 503 retries during container startup. + * Defaults to 120_000 (2 minutes). Should be at least as large as the sum of + * instanceGetTimeoutMS + portReadyTimeoutMS to avoid the client giving up + * before the container has finished starting. */ + retryTimeoutMs?: number; } /** @@ -74,4 +80,9 @@ export interface ITransport { * Check if connected (always true for HTTP) */ isConnected(): boolean; + + /** + * Update the 503 retry budget without recreating the transport + */ + setRetryTimeoutMs(ms: number): void; } diff --git a/packages/sandbox/src/clients/types.ts b/packages/sandbox/src/clients/types.ts index d3d098b3c..c3fdcc40c 100644 --- a/packages/sandbox/src/clients/types.ts +++ b/packages/sandbox/src/clients/types.ts @@ -53,6 +53,12 @@ export interface HttpClientOptions { * When provided, clients will use this transport instead of creating their own. */ transport?: ITransport; + + /** + * Total retry budget in milliseconds for 503 retries during container startup. + * Passed through to the transport layer. Defaults to 120_000 (2 minutes). + */ + retryTimeoutMs?: number; } /** diff --git a/packages/sandbox/src/sandbox.ts b/packages/sandbox/src/sandbox.ts index fad845382..73ec27d7f 100644 --- a/packages/sandbox/src/sandbox.ts +++ b/packages/sandbox/src/sandbox.ts @@ -316,6 +316,21 @@ export class Sandbox extends Container implements ISandbox { return (fn as (...a: unknown[]) => unknown).apply(client, args); } + /** + * Compute the transport retry budget from current container timeouts. + * + * The budget covers the full container startup window (instance provisioning + * + port readiness) plus a 30s margin for the maximum single backoff delay + * (capped at 30s in BaseTransport). The 120s floor preserves the previous + * default for short timeout configurations. + */ + private computeRetryTimeoutMs(): number { + const startupBudgetMs = + this.containerTimeouts.instanceGetTimeoutMS + + this.containerTimeouts.portReadyTimeoutMS; + return Math.max(120_000, startupBudgetMs + 30_000); + } + /** * Create a SandboxClient with current transport settings */ @@ -324,6 +339,7 @@ export class Sandbox extends Container implements ISandbox { logger: this.logger, port: 3000, stub: this, + retryTimeoutMs: this.computeRetryTimeoutMs(), ...(this.transport === 'websocket' && { transportMode: 'websocket' as const, wsUrl: 'ws://localhost:3000/ws' @@ -408,6 +424,8 @@ export class Sandbox extends Container implements ISandbox { ...this.containerTimeouts, ...storedTimeouts }; + // Update the transport retry budget to reflect stored timeouts + this.client.setRetryTimeoutMs(this.computeRetryTimeoutMs()); } }); } @@ -530,6 +548,9 @@ export class Sandbox extends Container implements ISandbox { // Persist to storage await this.ctx.storage.put('containerTimeouts', this.containerTimeouts); + // Update the transport retry budget to reflect new timeouts + this.client.setRetryTimeoutMs(this.computeRetryTimeoutMs()); + this.logger.debug('Container timeouts updated', this.containerTimeouts); }