Skip to content

Commit 4030b6f

Browse files
committed
improvement(sandbox): return a timed-out workbench run's output and the files it already wrote
A run_code call killed at its own timeout returned an empty stdout and only "timed out after Nms", so models reran long jobs from scratch. E2B's SDK throws a stream-less TimeoutError, which dropped the output the stream callbacks had already observed. - E2B adapter: a timed-out command returns the stdout/stderr its callbacks observed - Workbench runs (session sandboxes) that hit their own timeout throw a timeout abort carrying the output tail (64 KB) and the files already in the call's output directory; every existing timeout check still accepts it - The route returns that output in stdout with a 408 and says the workbench and its files persist, so the next call can rerun only the remaining work - Output goes through the same session provenance check and secret-name reporting as any other result; one-shot sandboxes and billing are unchanged
1 parent 395a394 commit 4030b6f

8 files changed

Lines changed: 341 additions & 20 deletions

File tree

‎apps/sim/lib/execution/remote-sandbox/e2b-session.test.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -613,6 +613,36 @@ describe('E2B session recovery', () => {
613613
expect(stopCommand).toHaveBeenCalledOnce()
614614
expect(killSandbox).not.toHaveBeenCalled()
615615
})
616+
617+
it('returns what a session command printed before its own timeout stopped it', async () => {
618+
runCommand.mockImplementation(
619+
async (
620+
_command: string,
621+
options: { onStdout?: (chunk: string) => void; onStderr?: (chunk: string) => void }
622+
) => {
623+
options.onStdout?.('rendered 1 of 3\n')
624+
options.onStderr?.('slow frame\n')
625+
options.onStdout?.('rendered 2 of 3\n')
626+
return {
627+
pid: 7,
628+
wait: async () => {
629+
throw Object.assign(
630+
new Error("deadline_exceeded: This error is likely due to exceeding 'timeoutMs'"),
631+
{ name: 'TimeoutError' }
632+
)
633+
},
634+
}
635+
}
636+
)
637+
const sandbox = await sessionSandbox('reconnected')
638+
const result = await sandbox?.runCommand('render', { timeoutMs: 1000 })
639+
expect(result).toMatchObject({
640+
timedOut: true,
641+
exitCode: 124,
642+
stdout: 'rendered 1 of 3\nrendered 2 of 3\n',
643+
stderr: 'slow frame\n',
644+
})
645+
})
616646
})
617647

618648
describe('E2B session lease', () => {

‎apps/sim/lib/execution/remote-sandbox/e2b.ts‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,14 @@ export async function stopE2BSessionProcess(
373373
if (sandbox) await stopSessionCommand(sandbox, process.id, signal)
374374
}
375375

376+
/** Joins what the stream callbacks observed with the streams the SDK returned, without repeating either. */
377+
function mergeObservedOutput(observed: string, returned: string): string {
378+
if (!observed) return returned
379+
if (!returned || observed.endsWith(returned)) return observed
380+
if (returned.startsWith(observed)) return returned
381+
return observed + returned
382+
}
383+
376384
class E2BSandboxHandle implements SandboxHandle {
377385
private killed = false
378386
private killPromise: Promise<void> | null = null
@@ -676,12 +684,6 @@ class E2BSandboxHandle implements SandboxHandle {
676684
// process handle because `background: true` is fixed above.
677685
result = started
678686
}
679-
const mergeObservedOutput = (observed: string, returned: string): string => {
680-
if (!observed) return returned
681-
if (!returned || observed.endsWith(returned)) return observed
682-
if (returned.startsWith(observed)) return returned
683-
return observed + returned
684-
}
685687
const stdout = mergeObservedOutput(observedStdout, result.stdout ?? '')
686688
const stderr = mergeObservedOutput(observedStderr, result.stderr ?? '')
687689
assertSandboxProcessOutputWithinLimit([stdout, stderr], options.maxOutputBytes)
@@ -727,16 +729,21 @@ class E2BSandboxHandle implements SandboxHandle {
727729
)
728730
const tailIfStreamed = (value: string | undefined, retain: boolean) =>
729731
retain || value === undefined ? value : tailStreamedSandboxOutput(value)
730-
const failureStdout = tailIfStreamed(failure.stdout, retainStdout)
731-
const failureStderr = tailIfStreamed(failure.stderr, retainStderr)
732732
if (isE2BExecutionTimeout(error)) {
733+
// The SDK's timeout error carries no streams; the callbacks observed the run's output.
734+
const stdout = mergeObservedOutput(observedStdout, failure.stdout ?? '')
735+
const stderr = mergeObservedOutput(observedStderr, failure.stderr ?? '')
733736
return {
734-
stdout: failureStdout ?? '',
735-
stderr: failureStderr ?? failure.message ?? '',
737+
stdout: retainStdout ? stdout : tailStreamedSandboxOutput(stdout),
738+
stderr:
739+
(retainStderr ? stderr : tailStreamedSandboxOutput(stderr)) ||
740+
`Command timed out after ${options.timeoutMs}ms`,
736741
exitCode: 124,
737742
timedOut: true,
738743
}
739744
}
745+
const failureStdout = tailIfStreamed(failure.stdout, retainStdout)
746+
const failureStderr = tailIfStreamed(failure.stderr, retainStderr)
740747
return {
741748
stdout: failureStdout ?? '',
742749
stderr: failureStderr ?? failure.message ?? getErrorMessage(error),

‎apps/sim/lib/execution/remote-sandbox/index.ts‎

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
isTimeoutAbortReason,
1010
} from '@/lib/core/execution-limits'
1111
import { recordSandboxTeardownFailure } from '@/lib/core/execution-limits/metrics'
12+
import { withinDeadline } from '@/lib/core/utils/deadline'
1213
import { buildJavaScriptRuntimeBindingsSource } from '@/lib/execution/code-placeholders/javascript-runtime'
1314
import { SANDBOX_SYSTEM_PATH } from '@/lib/execution/remote-sandbox/cli-tools.server'
1415
import { describeProcessExit } from '@/lib/execution/remote-sandbox/code-failure'
@@ -65,6 +66,10 @@ import {
6566
recordSessionFileInput,
6667
} from '@/lib/execution/remote-sandbox/session-file-provenance'
6768
import { withSandboxSessionLock } from '@/lib/execution/remote-sandbox/session-lock'
69+
import {
70+
SandboxExecutionTimeoutError,
71+
sandboxTimeoutProgress,
72+
} from '@/lib/execution/remote-sandbox/timeout'
6873
import type {
6974
CreateSandboxOptions,
7075
SandboxCodeResult,
@@ -258,6 +263,36 @@ function throwIfSandboxTimedOut(result: { timedOut?: boolean }): void {
258263
if (result.timedOut) throw new DOMException('timeout', 'AbortError')
259264
}
260265

266+
/** Bounds the output-directory listing a timeout report adds; a slow listing reports no files. */
267+
const TIMED_OUT_OUTPUT_LISTING_MS = 3_000
268+
269+
/**
270+
* A workbench run stopped by its own timeout reports what it produced, because the workbench and
271+
* its files outlive the call and the next call can resume from them. A one-shot sandbox is torn
272+
* down with everything in it, so it keeps the plain timeout.
273+
*/
274+
async function throwIfRunTimedOut(
275+
result: { timedOut?: boolean },
276+
streams: Array<string | undefined>,
277+
run: {
278+
sandbox: SandboxHandle
279+
lease: SandboxLease
280+
outputSandboxDir: string | undefined
281+
timeoutMs: number
282+
}
283+
): Promise<void> {
284+
if (!result.timedOut) return
285+
if (!run.lease.session) throw new DOMException('timeout', 'AbortError')
286+
const { outputSandboxDir, sandbox } = run
287+
const listed = outputSandboxDir
288+
? await withinDeadline(
289+
() => sandbox.listFiles(outputSandboxDir, { depth: SANDBOX_OUTPUT_DIR_MAX_DEPTH }),
290+
Date.now() + TIMED_OUT_OUTPUT_LISTING_MS
291+
).catch(() => [])
292+
: []
293+
throw new SandboxExecutionTimeoutError(sandboxTimeoutProgress(streams, run.timeoutMs, listed))
294+
}
295+
261296
function bindSandboxAbort(
262297
sandbox: SandboxHandle,
263298
provider: SandboxProviderId,
@@ -969,10 +1004,11 @@ async function executeInSandboxWithinBudget(
9691004
Object.keys(privateInputFiles.environment).length > 0
9701005

9711006
let execution: SandboxCodeResult
1007+
const runTimeoutMs = codeBudgetMs(req, signal)
9721008
try {
9731009
const endRun = phases.start('run')
9741010
execution = await sandbox.runCode(code, {
975-
timeoutMs: codeBudgetMs(req, signal),
1011+
timeoutMs: runTimeoutMs,
9761012
javascriptPreload: buildJavaScriptRuntimeBindingsSource(req.runtimeBindings ?? []),
9771013
maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
9781014
signal,
@@ -984,7 +1020,12 @@ async function executeInSandboxWithinBudget(
9841020
throw error
9851021
}
9861022
throwIfAborted(signal)
987-
throwIfSandboxTimedOut(execution)
1023+
await throwIfRunTimedOut(execution, [execution.text, execution.stdout, execution.stderr], {
1024+
sandbox,
1025+
lease,
1026+
outputSandboxDir: req.outputSandboxDir,
1027+
timeoutMs: runTimeoutMs,
1028+
})
9881029

9891030
if (execution.error) {
9901031
const errorMessage = `${execution.error.name}: ${execution.error.value}`
@@ -1166,6 +1207,7 @@ async function executeShellInSandboxWithinBudget(
11661207
endPrepare()
11671208

11681209
let result: SandboxCommandResult
1210+
const runTimeoutMs = codeBudgetMs(req, signal)
11691211
try {
11701212
const endRun = phases.start('run')
11711213
result = await sandbox.runCommand(code, {
@@ -1177,7 +1219,7 @@ async function executeShellInSandboxWithinBudget(
11771219
...privateInputFiles.environment,
11781220
...(req.session && req.outputSandboxDir ? { SIM_OUTPUT_DIR: req.outputSandboxDir } : {}),
11791221
},
1180-
timeoutMs: codeBudgetMs(req, signal),
1222+
timeoutMs: runTimeoutMs,
11811223
maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
11821224
signal,
11831225
rootUser: !lease.session,
@@ -1189,7 +1231,12 @@ async function executeShellInSandboxWithinBudget(
11891231
throw error
11901232
}
11911233
throwIfAborted(signal)
1192-
throwIfSandboxTimedOut(result)
1234+
await throwIfRunTimedOut(result, [result.stdout, result.stderr], {
1235+
sandbox,
1236+
lease,
1237+
outputSandboxDir: req.outputSandboxDir,
1238+
timeoutMs: runTimeoutMs,
1239+
})
11931240

11941241
const stdout = [result.stdout, result.stderr].filter(Boolean).join('\n')
11951242

‎apps/sim/lib/execution/remote-sandbox/output-limits.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,15 +48,16 @@ const STREAMED_OUTPUT_TRUNCATION_NOTE =
4848
*/
4949
export function tailStreamedSandboxOutput(
5050
value: string | undefined,
51-
limitBytes = MAX_SANDBOX_STREAMED_OUTPUT_TAIL_BYTES
51+
limitBytes = MAX_SANDBOX_STREAMED_OUTPUT_TAIL_BYTES,
52+
truncationNote = STREAMED_OUTPUT_TRUNCATION_NOTE
5253
): string {
5354
if (!value) return ''
5455
const buffer = Buffer.from(value, 'utf8')
5556
if (buffer.length <= limitBytes) return value
5657

5758
let start = buffer.length - limitBytes
5859
while (start < buffer.length && (buffer[start] & 0xc0) === 0x80) start += 1
59-
return `${STREAMED_OUTPUT_TRUNCATION_NOTE}${buffer.subarray(start).toString('utf8')}`
60+
return `${truncationNote}${buffer.subarray(start).toString('utf8')}`
6061
}
6162

6263
/**

‎apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import type {
1919
SandboxCommandResult,
2020
SandboxHandle,
2121
SandboxProvider,
22+
SandboxSessionRequest,
2223
} from '@/lib/execution/remote-sandbox/types'
2324

2425
const { mockCreate, mockFindSessionSandbox, mockResolveWorkspaceSandbox } = vi.hoisted(() => ({
@@ -41,6 +42,7 @@ vi.mock('@/lib/core/execution-limits/metrics', () => ({
4142
recordSandboxProviderLimit: vi.fn(),
4243
}))
4344

45+
import { isTimeoutAbortReason } from '@/lib/core/execution-limits'
4446
import {
4547
executeInSandbox,
4648
executeShellInSandbox,
@@ -56,6 +58,10 @@ import {
5658
recordSessionFileInput,
5759
} from '@/lib/execution/remote-sandbox/session-file-provenance'
5860
import { writeSessionSandboxFile } from '@/lib/execution/remote-sandbox/session-files'
61+
import {
62+
MAX_SANDBOX_TIMEOUT_OUTPUT_TAIL_BYTES,
63+
SandboxExecutionTimeoutError,
64+
} from '@/lib/execution/remote-sandbox/timeout'
5965

6066
remoteSandboxProviderMockFns.mockResolveProvider.mockImplementation(
6167
(): SandboxProvider => ({
@@ -941,6 +947,122 @@ describe('session sandbox lease', () => {
941947
expect(calls.killed).toBe(false)
942948
})
943949

950+
describe('a workbench run stopped by its own timeout', () => {
951+
const OUTPUT_DIR = '/tmp/sim/outputs/call-timeout'
952+
953+
async function timedOutRun(
954+
kind: 'code' | 'shell',
955+
streams: { stdout: string; stderr: string },
956+
listFiles: SandboxHandle['listFiles'],
957+
acceptOutputProvenance?: SandboxSessionRequest['acceptOutputProvenance']
958+
): Promise<unknown> {
959+
const { handle } = fakeSandbox(`timeout-${kind}`)
960+
mockFindSessionSandbox.mockResolvedValue(handle)
961+
handle.runCode = async () => ({ text: '', ...streams, timedOut: true })
962+
handle.runCommand = async () => ({ ...streams, exitCode: 124, timedOut: true })
963+
handle.listFiles = listFiles
964+
const request = {
965+
...CODE_REQUEST,
966+
session: {
967+
key: `timeout-${kind}`,
968+
...(acceptOutputProvenance ? { acceptOutputProvenance } : {}),
969+
},
970+
outputSandboxDir: OUTPUT_DIR,
971+
}
972+
return (
973+
kind === 'code'
974+
? executeInSandbox(request)
975+
: executeShellInSandbox({ ...request, envs: {} })
976+
).then(
977+
() => {
978+
throw new Error('Expected the run to time out')
979+
},
980+
(error: unknown) => error
981+
)
982+
}
983+
984+
it.each(['code', 'shell'] as const)(
985+
'reports the %s output so far and the files already exported',
986+
async (kind) => {
987+
const error = await timedOutRun(
988+
kind,
989+
{ stdout: 'rendered 1 of 3\nrendered 2 of 3', stderr: 'slow frame' },
990+
async () => [
991+
{
992+
path: `${OUTPUT_DIR}/${SANDBOX_OUTPUT_DIR_SENTINEL}`,
993+
relativePath: SANDBOX_OUTPUT_DIR_SENTINEL,
994+
kind: 'file',
995+
size: 0,
996+
},
997+
{ path: `${OUTPUT_DIR}/frames`, relativePath: 'frames', kind: 'directory', size: 0 },
998+
{
999+
path: `${OUTPUT_DIR}/frames/2.png`,
1000+
relativePath: 'frames/2.png',
1001+
kind: 'file',
1002+
size: 9,
1003+
},
1004+
{
1005+
path: `${OUTPUT_DIR}/frames/1.png`,
1006+
relativePath: 'frames/1.png',
1007+
kind: 'file',
1008+
size: 9,
1009+
},
1010+
]
1011+
)
1012+
expect(isTimeoutAbortReason(error)).toBe(true)
1013+
if (!(error instanceof SandboxExecutionTimeoutError)) throw error
1014+
expect(error.progress.output).toBe('rendered 1 of 3\nrendered 2 of 3\nslow frame')
1015+
expect(error.progress.outputFiles).toEqual([
1016+
`${OUTPUT_DIR}/frames/1.png`,
1017+
`${OUTPUT_DIR}/frames/2.png`,
1018+
])
1019+
expect(error.progress.stoppedAfterMs).toBeGreaterThan(0)
1020+
expect(error.progress.stoppedAfterMs).toBeLessThanOrEqual(CODE_REQUEST.timeoutMs)
1021+
}
1022+
)
1023+
1024+
it.each([
1025+
['fails', async () => Promise.reject(new Error('listing unavailable'))],
1026+
['never answers', () => new Promise<never>(() => {})],
1027+
] as const)('still reports the output when the file listing %s', async (_case, listFiles) => {
1028+
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })
1029+
try {
1030+
const pending = timedOutRun('shell', { stdout: 'step 1', stderr: '' }, listFiles)
1031+
await vi.runAllTimersAsync()
1032+
const error = await pending
1033+
if (!(error instanceof SandboxExecutionTimeoutError)) throw error
1034+
expect(error.progress.output).toBe('step 1')
1035+
expect(error.progress.outputFiles).toEqual([])
1036+
} finally {
1037+
vi.useRealTimers()
1038+
}
1039+
})
1040+
1041+
it('withholds the output when the workbench secret provenance is unavailable', async () => {
1042+
const error = await timedOutRun(
1043+
'shell',
1044+
{ stdout: 'token=secret-value', stderr: '' },
1045+
async () => [],
1046+
async () => {
1047+
throw new Error('Workbench output withheld because its secret provenance is unavailable')
1048+
}
1049+
)
1050+
expect(error).not.toBeInstanceOf(SandboxExecutionTimeoutError)
1051+
expect(getErrorMessage(error)).not.toContain('secret-value')
1052+
})
1053+
1054+
it('keeps only the end of an output too large to return', async () => {
1055+
const lines = Array.from({ length: 20_000 }, (_, index) => `frame ${index}`).join('\n')
1056+
const error = await timedOutRun('shell', { stdout: lines, stderr: '' }, async () => [])
1057+
if (!(error instanceof SandboxExecutionTimeoutError)) throw error
1058+
expect(Buffer.byteLength(error.progress.output)).toBeLessThanOrEqual(
1059+
MAX_SANDBOX_TIMEOUT_OUTPUT_TAIL_BYTES + 64
1060+
)
1061+
expect(error.progress.output.endsWith('frame 19999')).toBe(true)
1062+
expect(error.progress.output.startsWith('[earlier output omitted]')).toBe(true)
1063+
})
1064+
})
1065+
9441066
it('keeps the one-shot teardown when no session is requested', async () => {
9451067
const { handle, calls } = fakeSandbox('sb-oneshot')
9461068
mockCreate.mockResolvedValue(handle)

0 commit comments

Comments
 (0)