Skip to content

Commit 7b942ca

Browse files
committed
improvement(sandbox): verify the workbench CLI inside each command's launch
Every warm workbench call ran a separate CLI integrity check command (ownership write, launch, stop receipt, settle) before the user's command, about a third of a warm call's fixed cost. - The session supervisor's PID 1 now runs the same check as the ordinary user, after recording itself and before starting the command; a mismatch starts nothing and is reported on the stop receipt the call already fetches - A refused launch is repaired under the session lock and relaunched once; a second refusal fails the call - The check runs with an isolated interpreter instead of the user's login shell, never follows links or blocks on non-regular files, and treats an unreadable artifact as stale so repair overwrites it - A reused machine runs no separate check command; a new machine still installs the CLI first - Gated live test against real E2B covers refusal, repair, hooks, FIFOs, Stop during a slow check, and the warm-call cost
1 parent 395a394 commit 7b942ca

10 files changed

Lines changed: 670 additions & 63 deletions

File tree

‎apps/sim/lib/execution/remote-sandbox/e2b-session.test.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ import {
5151
stopE2BSessionProcess,
5252
} from '@/lib/execution/remote-sandbox/e2b'
5353
import { observeSandboxExecution } from '@/lib/execution/remote-sandbox/observer'
54+
import { SessionCliStaleError } from '@/lib/execution/remote-sandbox/session-cli'
5455

5556
setEnv({ E2B_API_KEY: 'test', MOTHERSHIP_E2B_TEMPLATE_ID: 'mothership-template' })
5657

@@ -613,6 +614,37 @@ describe('E2B session recovery', () => {
613614
expect(stopCommand).toHaveBeenCalledOnce()
614615
expect(killSandbox).not.toHaveBeenCalled()
615616
})
617+
618+
it.each([
619+
['stale', SessionCliStaleError],
620+
['failed', Error],
621+
] as const)(
622+
'rejects a launch whose receipt reports a %s CLI, whatever its process returned',
623+
async (cli, expected) => {
624+
runCommand.mockResolvedValue({
625+
pid: 9,
626+
wait: async () => ({ stdout: '', stderr: '', exitCode: 0 }),
627+
})
628+
stopCommand.mockResolvedValue({
629+
stdout: JSON.stringify({ settled: true, cli }),
630+
stderr: '',
631+
exitCode: 0,
632+
})
633+
const sandbox = await sessionSandbox('reconnected')
634+
const launched = sandbox?.runCommand('touch /tmp/marker', {
635+
timeoutMs: 1000,
636+
verifyCli: { path: '/home/user/.sim-cli/r/cli.mjs', artifacts: { a: 'b' } },
637+
})
638+
await expect(launched).rejects.toBeInstanceOf(expected)
639+
await expect(launched).rejects.toThrow(cli === 'stale' ? 'changed' : 'verification failed')
640+
const [command] = runCommand.mock.calls.at(-1) ?? []
641+
expect(command).toContain(
642+
Buffer.from(
643+
JSON.stringify({ path: '/home/user/.sim-cli/r/cli.mjs', artifacts: { a: 'b' } })
644+
).toString('base64')
645+
)
646+
}
647+
)
616648
})
617649

618650
describe('E2B session lease', () => {

‎apps/sim/lib/execution/remote-sandbox/e2b.ts‎

Lines changed: 49 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,10 @@ import {
5555
tailStreamedSandboxOutput,
5656
} from '@/lib/execution/remote-sandbox/output-limits'
5757
import { resolveSandboxDirectoryEntryPath } from '@/lib/execution/remote-sandbox/paths'
58+
import {
59+
SessionCliStaleError,
60+
SessionCliVerificationError,
61+
} from '@/lib/execution/remote-sandbox/session-cli'
5862
import { withSandboxSessionLock } from '@/lib/execution/remote-sandbox/session-lock'
5963
import {
6064
type SessionProcessIdentity,
@@ -82,6 +86,7 @@ import type {
8286
SandboxImageMaterialization,
8387
SandboxKind,
8488
SandboxProvider,
89+
SessionCliVerification,
8590
} from '@/lib/execution/remote-sandbox/types'
8691

8792
const logger = createLogger('E2BSandboxProvider')
@@ -319,11 +324,17 @@ async function listAllSandboxes(paginator: SandboxPaginator): Promise<SandboxInf
319324
return sandboxes
320325
}
321326

327+
/** What a session command's launch found when it verified the deployment CLI; absent when it matched. */
328+
interface SessionCommandReceipt {
329+
cli?: 'stale' | 'failed'
330+
}
331+
332+
/** Settles the command; the receipt is the only proof that a refused launch started nothing. */
322333
async function stopSessionCommand(
323334
sandbox: E2BSandbox,
324335
processId: string,
325336
signal = AbortSignal.timeout(10_000)
326-
): Promise<void> {
337+
): Promise<SessionCommandReceipt> {
327338
signal.throwIfAborted()
328339
const result = await sandbox.commands.run(sessionProcessStopCommand(processId), {
329340
user: 'root',
@@ -335,6 +346,7 @@ async function stopSessionCommand(
335346
if (result.exitCode !== 0 || !isRecordLike(receipt) || receipt.settled !== true) {
336347
throw new Error('Sandbox command namespace exit is unconfirmed')
337348
}
349+
return receipt.cli === 'stale' || receipt.cli === 'failed' ? { cli: receipt.cli } : {}
338350
}
339351

340352
/** Recovers the exact recorded workbench; expiry is absence, while lookup failure stays unresolved. */
@@ -405,6 +417,10 @@ class E2BSandboxHandle implements SandboxHandle {
405417
return this.sandbox.sandboxId
406418
}
407419

420+
get verifiesCliAtLaunch(): boolean {
421+
return this.sessionKey !== undefined
422+
}
423+
408424
outlives(lifetimeMs: number, fromMs: number): boolean {
409425
return (
410426
this.sessionDeadlineAtMs !== undefined &&
@@ -433,6 +449,7 @@ class E2BSandboxHandle implements SandboxHandle {
433449
javascriptPreload?: string
434450
maxOutputBytes?: number
435451
signal?: AbortSignal
452+
verifyCli?: SessionCliVerification
436453
}
437454
): Promise<SandboxCodeResult> {
438455
const runId = generateShortId(12)
@@ -459,6 +476,7 @@ class E2BSandboxHandle implements SandboxHandle {
459476
maxOutputBytes: options.maxOutputBytes,
460477
signal: options.signal,
461478
atMostOnce: true,
479+
verifyCli: options.verifyCli,
462480
envs: {
463481
...options.envs,
464482
SIM_CODE_PATH: codePath,
@@ -511,6 +529,25 @@ class E2BSandboxHandle implements SandboxHandle {
511529
command: string,
512530
options: RunCommandOptions,
513531
operation: 'code' | 'command'
532+
): Promise<SandboxCommandResult> {
533+
let refusal: SessionCommandReceipt['cli']
534+
const outcome = await this.runOwnedCommand(command, options, operation, (receipt) => {
535+
refusal = receipt.cli
536+
}).then(
537+
(result) => ({ result }),
538+
(error: unknown) => ({ error })
539+
)
540+
if (refusal === 'stale') throw new SessionCliStaleError()
541+
if (refusal === 'failed') throw new SessionCliVerificationError()
542+
if ('error' in outcome) throw outcome.error
543+
return outcome.result
544+
}
545+
546+
private async runOwnedCommand(
547+
command: string,
548+
options: RunCommandOptions,
549+
operation: 'code' | 'command',
550+
onReceipt: (receipt: SessionCommandReceipt) => void
514551
): Promise<SandboxCommandResult> {
515552
if (this.sessionKey !== undefined) options.signal?.throwIfAborted()
516553
let commandDeadlineAtMs: number | undefined
@@ -529,9 +566,9 @@ class E2BSandboxHandle implements SandboxHandle {
529566
let observedStderr = ''
530567
const processId = this.sessionKey !== undefined ? generateId() : undefined
531568
let processRecorded = false
532-
let stopping: Promise<void> | undefined
533-
const stopCommand = (): Promise<void> => {
534-
if (!processId || !processRecorded) return Promise.resolve()
569+
let stopping: Promise<SessionCommandReceipt> | undefined
570+
const stopCommand = (): Promise<SessionCommandReceipt | undefined> => {
571+
if (!processId || !processRecorded) return Promise.resolve(undefined)
535572
stopping ??= stopSessionCommand(this.sandbox, processId).catch((error: unknown) => {
536573
logger.warn('Failed to stop sandbox process', {
537574
sandboxId: this.sandboxId,
@@ -587,7 +624,12 @@ class E2BSandboxHandle implements SandboxHandle {
587624
commandDeadlineAtMs = Date.now() + processOptions.timeoutMs
588625
started = await this.sandbox.commands.run(
589626
processId
590-
? sessionProcessCommand(processId, prepared.command, options.rootUser)
627+
? sessionProcessCommand(
628+
processId,
629+
prepared.command,
630+
options.rootUser,
631+
options.verifyCli
632+
)
591633
: prepared.command,
592634
{
593635
...processOptions,
@@ -746,11 +788,12 @@ class E2BSandboxHandle implements SandboxHandle {
746788
options.signal?.removeEventListener('abort', onAbort)
747789
/** The named SDK process can exit before its namespace; only the kernel receipt settles it. */
748790
if (processId && processRecorded) {
749-
await stopCommand().catch((error: unknown) => {
791+
const receipt = await stopCommand().catch((error: unknown) => {
750792
reportUnsettledSandboxProcess(processId)
751793
throw error
752794
})
753795
await settleSandboxProcess(processId)
796+
if (receipt) onReceipt(receipt)
754797
}
755798
}
756799
}

‎apps/sim/lib/execution/remote-sandbox/index.ts‎

Lines changed: 82 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,12 @@ import {
5959
ensureSessionSandbox,
6060
SESSION_SANDBOX_IDLE_MS,
6161
} from '@/lib/execution/remote-sandbox/session'
62-
import { sessionCommandPath } from '@/lib/execution/remote-sandbox/session-cli'
62+
import {
63+
ensureSessionCli,
64+
SessionCliStaleError,
65+
sessionCliVerification,
66+
sessionCommandPath,
67+
} from '@/lib/execution/remote-sandbox/session-cli'
6368
import {
6469
readSessionSecretProvenance,
6570
recordSessionFileInput,
@@ -82,6 +87,7 @@ import type {
8287
SandboxProviderId,
8388
SandboxSessionRequest,
8489
SandboxShellExecutionRequest,
90+
SessionCliVerification,
8591
} from '@/lib/execution/remote-sandbox/types'
8692

8793
export type {
@@ -254,6 +260,42 @@ async function withSandboxExecutionBudget<T>(
254260
}
255261
}
256262

263+
/**
264+
* Launches a session command that verifies the deployment CLI as it starts. A launch that found
265+
* the CLI changed started nothing, so the CLI is repaired under the session lock and the command
266+
* launched once more; a second refusal means something keeps changing it, and the call fails.
267+
*/
268+
async function launchWithVerifiedCli<T>(
269+
sandbox: SandboxHandle,
270+
lease: SandboxLease,
271+
session: SandboxSessionRequest | undefined,
272+
signal: AbortSignal,
273+
phases: SandboxPhaseTimer,
274+
launch: (verifyCli: SessionCliVerification | undefined) => Promise<T>
275+
): Promise<T> {
276+
const cli = lease.session && sandbox.verifiesCliAtLaunch ? session?.cli : undefined
277+
if (!session || !cli) return launch(undefined)
278+
const verifyCli = sessionCliVerification(cli)
279+
try {
280+
return await launch(verifyCli)
281+
} catch (error) {
282+
if (!(error instanceof SessionCliStaleError)) throw error
283+
}
284+
const endCli = phases.start('cli')
285+
await withSandboxSessionLock(session.key, signal, (lockSignal) =>
286+
ensureSessionCli(sandbox, cli, lockSignal, remainingSandboxBudgetMs(signal))
287+
)
288+
endCli()
289+
try {
290+
return await launch(verifyCli)
291+
} catch (error) {
292+
if (error instanceof SessionCliStaleError) {
293+
throw new Error('Workbench CLI installation could not be verified')
294+
}
295+
throw error
296+
}
297+
}
298+
257299
function throwIfSandboxTimedOut(result: { timedOut?: boolean }): void {
258300
if (result.timedOut) throw new DOMException('timeout', 'AbortError')
259301
}
@@ -971,13 +1013,22 @@ async function executeInSandboxWithinBudget(
9711013
let execution: SandboxCodeResult
9721014
try {
9731015
const endRun = phases.start('run')
974-
execution = await sandbox.runCode(code, {
975-
timeoutMs: codeBudgetMs(req, signal),
976-
javascriptPreload: buildJavaScriptRuntimeBindingsSource(req.runtimeBindings ?? []),
977-
maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
1016+
execution = await launchWithVerifiedCli(
1017+
sandbox,
1018+
lease,
1019+
req.session,
9781020
signal,
979-
...(hasExecutionEnvironment ? { envs: executionEnvironment } : {}),
980-
})
1021+
phases,
1022+
(verifyCli) =>
1023+
sandbox.runCode(code, {
1024+
timeoutMs: codeBudgetMs(req, signal),
1025+
javascriptPreload: buildJavaScriptRuntimeBindingsSource(req.runtimeBindings ?? []),
1026+
maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
1027+
signal,
1028+
...(hasExecutionEnvironment ? { envs: executionEnvironment } : {}),
1029+
verifyCli,
1030+
})
1031+
)
9811032
endRun()
9821033
} catch (error) {
9831034
throwIfAborted(signal)
@@ -1168,21 +1219,31 @@ async function executeShellInSandboxWithinBudget(
11681219
let result: SandboxCommandResult
11691220
try {
11701221
const endRun = phases.start('run')
1171-
result = await sandbox.runCommand(code, {
1172-
envs: {
1173-
...selected?.envs,
1174-
...envs,
1175-
...req.session?.envs,
1176-
PATH: sessionCommandPath(req.session, selected?.envs?.PATH ?? SANDBOX_SYSTEM_PATH),
1177-
...privateInputFiles.environment,
1178-
...(req.session && req.outputSandboxDir ? { SIM_OUTPUT_DIR: req.outputSandboxDir } : {}),
1179-
},
1180-
timeoutMs: codeBudgetMs(req, signal),
1181-
maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
1222+
const commandEnvironment = {
1223+
...selected?.envs,
1224+
...envs,
1225+
...req.session?.envs,
1226+
PATH: sessionCommandPath(req.session, selected?.envs?.PATH ?? SANDBOX_SYSTEM_PATH),
1227+
...privateInputFiles.environment,
1228+
...(req.session && req.outputSandboxDir ? { SIM_OUTPUT_DIR: req.outputSandboxDir } : {}),
1229+
}
1230+
result = await launchWithVerifiedCli(
1231+
sandbox,
1232+
lease,
1233+
req.session,
11821234
signal,
1183-
rootUser: !lease.session,
1184-
atMostOnce: true,
1185-
})
1235+
phases,
1236+
(verifyCli) =>
1237+
sandbox.runCommand(code, {
1238+
envs: commandEnvironment,
1239+
timeoutMs: codeBudgetMs(req, signal),
1240+
maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
1241+
signal,
1242+
rootUser: !lease.session,
1243+
atMostOnce: true,
1244+
verifyCli,
1245+
})
1246+
)
11861247
endRun()
11871248
} catch (error) {
11881249
throwIfAborted(signal)

‎apps/sim/lib/execution/remote-sandbox/phase-timing.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
/**
22
* Phases of one sandbox execution, in the order they run. `lease` spans the whole acquisition
3-
* (session lock included); for a session, `lookup` and `cli` are the parts of it spent finding or
4-
* creating the machine and verifying its deployment tooling.
3+
* (session lock included); for a session, `lookup` is the part of it spent finding or creating the
4+
* machine. `cli` is time spent installing or repairing deployment tooling: on a new machine, or
5+
* after a launch refused a changed CLI. A reused machine verifies it inside `run`.
56
*/
67
type SandboxPhase =
78
| 'resolve'

0 commit comments

Comments
 (0)