diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index b177f12d..4bd2d12b 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -320,6 +320,9 @@ jobs: VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} + SPARK_API_URL: ${{ secrets.SPARK_API_URL }} + CLOUDFLARE_WARP_CLIENT_ID: ${{ secrets.CLOUDFLARE_WARP_CLIENT_ID }} + CLOUDFLARE_WARP_CLIENT_SECRET: ${{ secrets.CLOUDFLARE_WARP_CLIENT_SECRET }} SUPABASE_CLI_STABLE_VERSION: ${{ needs.prepare.outputs.cli_stable_version }} SUPABASE_CLI_BETA_VERSION: ${{ needs.prepare.outputs.cli_beta_version }} SUPABASE_CLI_NEXT_VERSION: ${{ needs.prepare.outputs.cli_next_version }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1b725930..98dac992 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,6 +46,10 @@ Add a `*.experiment.ts` file under `experiments//` for the agent, model, The suite folder decides which evals the experiment runs: `experiments//` runs the evals in `evals//`. If this experiment should be part of our published benchmark, put it in `experiments/benchmark/` and include a corresponding `*-no-skills` variant (`skills: []`) to compare results with and without skills. To run one experiment in several suites, export a shared preset from `experiments/presets.ts` and spread it in a file under each suite folder. +## Spark evals + +Local runs need `SPARK_API_URL` and WARP access. For CI, configure `SPARK_API_URL`, `CLOUDFLARE_WARP_CLIENT_ID`, and `CLOUDFLARE_WARP_CLIENT_SECRET` as repository secrets alongside the existing Vercel and provider credentials. + ## Submitting evals for review Before submitting an eval for review, try running it locally to sanity check that it can complete without errors. It's okay if agents fail the eval, we just don't want them to be scored unfairly for framework limitations. diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index a64926d7..e7917374 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -530,6 +530,7 @@ async function runOne( ? disposable( await createBareSandbox({ env: session.env, + mountDockerSocket: session.mountDockerSocket, skills: skillSources, mounts: supabaseMcpServerMounts(), }) diff --git a/apps/framework/scripts/redact-spark-result.test.ts b/apps/framework/scripts/redact-spark-result.test.ts new file mode 100644 index 00000000..feca813a --- /dev/null +++ b/apps/framework/scripts/redact-spark-result.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from 'vitest'; +import { sanitizeResult } from './redact-spark-result.js'; + +// Dummy JWT signature is base64url for the word "signature". +const fakeAccessToken = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ4In0.c2lnbmF0dXJl'; +const fakeRefreshToken = 'opaque-refresh+/token='; + +describe('Spark result sanitization', () => { + it('redacts configured values, encoded copies and object keys', () => { + const secret = 'https://private.example/a?token=secret'; + const value = { [secret]: [secret, encodeURIComponent(secret), 4] }; + expect(JSON.parse(sanitizeResult(value, [secret]))).toEqual({ + '[REDACTED]': ['[REDACTED]', '[REDACTED]', 4], + }); + }); + + it.each(['structured', 'stdout', 'escaped'])( + 'discovers session tokens in %s responses and redacts every copy', + (format) => { + const session = { + access_token: fakeAccessToken, + refresh_token: fakeRefreshToken, + token_type: 'bearer', + }; + const response = + format === 'structured' + ? session + : format === 'stdout' + ? `signup response: ${JSON.stringify(session)}` + : JSON.stringify(JSON.stringify(session)); + const checks = [{ passed: false, notes: 'found: JWT-shaped token' }]; + const value = { + copies: [ + fakeAccessToken, + fakeRefreshToken, + encodeURIComponent(fakeRefreshToken), + ], + response, + checks, + }; + const payload = sanitizeResult(value, []); + for (const token of [fakeAccessToken, fakeRefreshToken]) { + expect(payload).not.toContain(token); + expect(payload).not.toContain(encodeURIComponent(token)); + } + expect(JSON.parse(payload)).toMatchObject({ + copies: ['[REDACTED]', '[REDACTED]', '[REDACTED]'], + checks, + }); + expect(payload).toContain('bearer'); + } + ); + + it('redacts JWTs printed without a session field', () => { + expect( + JSON.parse(sanitizeResult({ report: `Bearer ${fakeAccessToken}` }, [])) + ).toEqual({ + report: 'Bearer [REDACTED]', + }); + }); +}); diff --git a/apps/framework/scripts/redact-spark-result.ts b/apps/framework/scripts/redact-spark-result.ts new file mode 100644 index 00000000..d5f33676 --- /dev/null +++ b/apps/framework/scripts/redact-spark-result.ts @@ -0,0 +1,105 @@ +import { readFileSync, writeFileSync } from 'node:fs'; + +const JWT = /eyJ[\w-]+\.[\w-]+\.[\w-]+/g; +const SESSION_TOKEN = + /\\*["'](?:access_token|refresh_token|id_token)\\*["']\s*:\s*\\*["']([\w.~+/%=-]+)\\*["']/g; + +function sessionTokens(value: unknown): string[] { + if (typeof value === 'string') { + return [...value.matchAll(JWT)] + .map((match) => match[0]) + .concat([...value.matchAll(SESSION_TOKEN)].map((match) => match[1])); + } + if (Array.isArray(value)) return value.flatMap(sessionTokens); + if (value && typeof value === 'object') { + return Object.entries(value).flatMap(([key, item]) => [ + ...(['access_token', 'refresh_token', 'id_token'].includes(key) && + typeof item === 'string' + ? [item] + : []), + ...sessionTokens(key), + ...sessionTokens(item), + ]); + } + return []; +} + +function encodedSecret(secret: string): string { + return encodeURIComponent(secret).replace( + /[!'()*]/g, + (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}` + ); +} + +function redact(value: unknown, secrets: readonly string[]): unknown { + if (typeof value === 'string') { + let text = value; + for (const secret of secrets) { + text = text + .replaceAll(secret, '[REDACTED]') + .replaceAll(encodedSecret(secret), '[REDACTED]'); + } + return text; + } + if (Array.isArray(value)) return value.map((item) => redact(item, secrets)); + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [ + String(redact(key, secrets)), + redact(item, secrets), + ]) + ); + } + return value; +} + +export function sanitizeResult( + value: unknown, + configuredSecrets: readonly string[] +): string { + const secrets = [...new Set([...configuredSecrets, ...sessionTokens(value)])] + .filter(Boolean) + .sort((a, b) => b.length - a.length); + const payload = JSON.stringify(redact(value, secrets)); + if ( + secrets.some( + (secret) => + payload.includes(secret) || payload.includes(encodedSecret(secret)) + ) + ) { + throw new Error('Result still contains a private value'); + } + return payload; +} + +if (import.meta.main) { + try { + const secrets = [ + '/tmp/spark-warp-secrets.json', + '/tmp/spark-redactions.json', + ].flatMap((path) => { + const value: unknown = JSON.parse(readFileSync(path, 'utf8')); + if ( + !Array.isArray(value) || + !value.every((item): item is string => typeof item === 'string') + ) + throw new Error('Invalid Spark redactions'); + return value; + }); + const path = process.argv[2]; + const value: unknown = JSON.parse(readFileSync(path, 'utf8')); + writeFileSync(path, sanitizeResult(value, secrets)); + } catch (error) { + writeFileSync( + '/tmp/spark-redaction-error.log', + error instanceof Error + ? (error.stack ?? error.message) + : 'Redaction failed', + { mode: 0o600 } + ); + console.error( + 'Spark result redaction failed. Inspect private Sandbox logs.' + ); + process.exitCode = 1; + } +} diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts index e7cabc54..7ed7ba1c 100644 --- a/apps/framework/scripts/run-vercel-evals.test.ts +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -1,4 +1,5 @@ -import { APIError } from '@vercel/sandbox'; +import { APIError, Sandbox } from '@vercel/sandbox'; +import { AbortError } from 'p-retry'; import { resolveCliVersion, type CliChannel } from '@supabase-evals/sandbox'; import { execFileSync } from 'node:child_process'; import { @@ -29,15 +30,76 @@ import { requiredCliChannels, resolveChannelPins, runBounded, + runPairOnce, tagValue, expandJobs, type EvalPair, + sparkWarpMdm, } from './run-vercel-evals.js'; vi.mock('@supabase-evals/sandbox', () => ({ resolveCliVersion: vi.fn() })); const BROKERED_NAMES = BROKERED_KEYS.map(({ name }) => name); +it('escapes credentials as XML text in the WARP configuration', () => { + expect(sparkWarpMdm('client<&>', 'secret<&>')).toContain( + 'auth_client_idclient<&>' + ); + expect(sparkWarpMdm('client<&>', 'secret<&>')).toContain( + 'auth_client_secretsecret<&>' + ); +}); + +describe('Spark terminal failures', () => { + const options = { + pair: { + eval_id: 'eval-1', + experiment: 'experiment-1', + eval_suite: 'spark', + }, + pairs: [], + revision: 'test', + repoUrl: 'https://github.com/supabase/evals', + outputDir: '/tmp/unused-spark-results', + runs: 1, + timeoutSec: 1, + concurrency: 1, + vcpus: 1, + attempt: 1, + run: 1, + pins: {}, + }; + const credentials = { token: 'test', teamId: 'test', projectId: 'test' }; + + beforeEach(() => { + for (const name of BROKERED_NAMES) vi.stubEnv(name, ''); + vi.stubEnv('SPARK_API_URL', 'https://control.example'); + vi.stubEnv('CLOUDFLARE_WARP_CLIENT_ID', 'test.access'); + vi.stubEnv('CLOUDFLARE_WARP_CLIENT_SECRET', 'test'); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it.each(['unauthorized Sandbox', 'missing credentials'])( + 'aborts %s without retrying', + async (reason) => { + if (reason === 'missing credentials') + vi.stubEnv('CLOUDFLARE_WARP_CLIENT_SECRET', ''); + const create = vi + .spyOn(Sandbox, 'create') + .mockRejectedValue(new APIError(new Response(null, { status: 401 }))); + await expect(runPairOnce(options, credentials)).rejects.toBeInstanceOf( + AbortError + ); + expect(create).toHaveBeenCalledTimes( + reason === 'missing credentials' ? 0 : 1 + ); + } + ); +}); + describe('agentEnvironment', () => { const originalValues = new Map(); @@ -422,6 +484,27 @@ describe('Vercel eval controller', () => { } }); + it('downloads only sanitized JSON for Spark, never workspace or session archives', async () => { + const downloadFile = vi.fn().mockResolvedValue('downloaded'); + const result = await downloadResults( + { downloadFile }, + { eval_id: 'eval-1', experiment: 'experiment-1', eval_suite: 'spark' }, + 1, + '/tmp/unused-spark-results', + '/vercel/evals' + ); + expect(downloadFile).toHaveBeenCalledExactlyOnceWith( + { + path: '/vercel/evals/results/spark/experiment-1/eval-1/run-1/result.json', + }, + { path: result.partialPath }, + { mkdirRecursive: true } + ); + expect(result.partialPath).toBe( + '/tmp/unused-spark-results/raw-results-spark__experiment-1__eval-1/eval-1/run-1/result.json.partial' + ); + }); + it('downloads result metadata separately from agent workspace files', async () => { const temporary = mkdtempSync(join(tmpdir(), 'vercel-eval-test-')); const sandboxFiles = join(temporary, 'sandbox'); diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index aaf7318f..925f2ad3 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -22,6 +22,7 @@ import pLimit from 'p-limit'; import pRetry, { AbortError } from 'p-retry'; import { z } from 'zod'; import { positiveInteger, readFlag } from '../lib/cli-args.js'; +import { SPARK_PROXY_BYPASS } from '../../../experiments/spark/lib/runtime.js'; import { discoverExperimentFiles } from '../lib/experiment-files.js'; const ROOT = fileURLToPath(new URL('../../../', import.meta.url)); @@ -234,7 +235,11 @@ interface SandboxCommandOptions { } class SandboxCommandError extends Error { - constructor(step: string, exitCode: number, output: string) { + constructor( + readonly step: string, + exitCode: number, + output: string + ) { const detail = output.trim().slice(-4_000); super(`${step} exited with code ${exitCode}${detail ? `\n${detail}` : ''}`); this.name = 'SandboxCommandError'; @@ -333,7 +338,21 @@ async function runPairs(options: RunnerOptions): Promise { } } -async function runPairOnce( +export function sparkWarpMdm(clientId: string, clientSecret: string): string { + const entries = Object.entries({ + organization: 'supabase', + auth_client_id: clientId, + auth_client_secret: clientSecret, + service_mode: 'proxy', + warp_tunnel_protocol: 'masque', + }).map( + ([key, value]) => + `${key}${value.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>')}` + ); + return `${entries.join('')}proxy_port40000`; +} + +export async function runPairOnce( options: PairOptions, credentials: VercelCredentials ): Promise { @@ -341,12 +360,29 @@ async function runPairOnce( const label = jobLabel(pair, run); let sandbox: Sandbox | undefined; let pendingResult: PendingResult | undefined; + let sparkSetupInstalled = false; + const spark = pair.eval_suite === 'spark'; + const sparkConfig = spark + ? Object.fromEntries( + [ + 'CLOUDFLARE_WARP_CLIENT_ID', + 'CLOUDFLARE_WARP_CLIENT_SECRET', + 'SPARK_API_URL', + ].map((name) => { + const value = process.env[name]; + if (!value) throw new AbortError(`Missing ${name}`); + return [name, value]; + }) + ) + : {}; try { sandbox = await createSandbox(label, { ...credentials, name: sandboxName(pair, run), - runtime: 'node24', + ...(spark + ? { image: 'vercel/sandbox/universal' } + : { runtime: 'node24' }), source: { type: 'git', url: options.repoUrl, @@ -372,6 +408,7 @@ async function runPairOnce( `${label} attempt ${options.attempt}: ${SANDBOX_DASHBOARD_URL}/${sandbox.name}` ); + const repoCwd = spark ? `${sandbox.cwd}/evals` : sandbox.cwd; await runSandboxCommand(sandbox, label, 'initialize submodules', { cmd: 'git', args: [ @@ -382,29 +419,104 @@ async function runPairOnce( '--init', '--recursive', ], - cwd: sandbox.cwd, + cwd: repoCwd, timeoutMs: 2 * 60 * 1_000, }); - await runSandboxCommand(sandbox, label, 'install Docker', { - cmd: 'dnf', - args: ['install', '-y', '-q', 'docker'], - sudo: true, - timeoutMs: 3 * 60 * 1_000, - }); - await sandbox.runCommand({ - cmd: 'dockerd', - sudo: true, - detached: true, - }); - await runSandboxCommand(sandbox, label, 'start Docker', { - cmd: 'bash', - args: [ - '-c', - 'for i in $(seq 60); do docker info >/dev/null 2>&1 && chmod 666 /var/run/docker.sock && exit 0; sleep 1; done; echo "dockerd not ready" >&2; exit 1', - ], - sudo: true, - timeoutMs: 90_000, - }); + if (spark) { + await sandbox.writeFiles([ + { + path: '/tmp/redact-spark-result.ts', + content: readFileSync( + new URL('./redact-spark-result.ts', import.meta.url) + ), + }, + { + path: '/tmp/spark-warp-secrets.json', + content: Buffer.from( + JSON.stringify([ + ...Object.values(sparkConfig), + new URL(sparkConfig.SPARK_API_URL).hostname, + ]) + ), + mode: 0o600, + }, + { + path: '/tmp/spark-warp-mdm.xml', + content: Buffer.from( + sparkWarpMdm( + sparkConfig.CLOUDFLARE_WARP_CLIENT_ID, + sparkConfig.CLOUDFLARE_WARP_CLIENT_SECRET + ) + ), + mode: 0o600, + }, + ]); + await runSandboxCommand(sandbox, label, 'install Spark WARP proxy', { + cmd: 'bash', + args: [ + '-euo', + 'pipefail', + '-c', + readFileSync( + new URL('./setup-spark-warp.sh', import.meta.url), + 'utf8' + ), + ], + sudo: true, + timeoutMs: 10 * 60 * 1_000, + }); + sparkSetupInstalled = true; + for (const cmd of ['warp-svc', 'dockerd']) { + await sandbox.runCommand({ cmd, sudo: true, detached: true }); + } + await runSandboxCommand(sandbox, label, 'connect Spark WARP proxy', { + cmd: 'bash', + args: [ + '-euo', + 'pipefail', + '-c', + `umask 077 +exec >>/tmp/spark-setup.log 2>&1 +for attempt in {1..60}; do + warp-cli --accept-tos registration show >/dev/null 2>&1 && break + sleep 1 +done +warp-cli --accept-tos connect +for attempt in {1..60}; do + if warp-cli --accept-tos status | grep -F 'Status update: Connected' >/dev/null && docker info >/dev/null 2>&1; then + chmod 666 /var/run/docker.sock + exit 0 + fi + sleep 1 +done +echo 'WARP did not connect' >&2 +exit 1`, + ], + sudo: true, + timeoutMs: 3 * 60 * 1_000, + }); + } else { + await runSandboxCommand(sandbox, label, 'install Docker', { + cmd: 'dnf', + args: ['install', '-y', '-q', 'docker'], + sudo: true, + timeoutMs: 3 * 60 * 1_000, + }); + await sandbox.runCommand({ + cmd: 'dockerd', + sudo: true, + detached: true, + }); + await runSandboxCommand(sandbox, label, 'start Docker', { + cmd: 'bash', + args: [ + '-c', + 'for i in $(seq 60); do docker info >/dev/null 2>&1 && chmod 666 /var/run/docker.sock && exit 0; sleep 1; done; echo "dockerd not ready" >&2; exit 1', + ], + sudo: true, + timeoutMs: 90_000, + }); + } // Installs the pnpm version pinned by packageManager in the checked-out // root package.json so the two can't drift. await runSandboxCommand(sandbox, label, 'install pnpm', { @@ -413,21 +525,30 @@ async function runPairOnce( '-c', `npm install --global "$(node -p 'require("./package.json").packageManager')"`, ], - cwd: sandbox.cwd, + cwd: repoCwd, sudo: true, timeoutMs: 2 * 60 * 1_000, }); await runSandboxCommand(sandbox, label, 'install dependencies', { cmd: 'pnpm', args: ['install', '--frozen-lockfile'], - cwd: sandbox.cwd, + cwd: repoCwd, timeoutMs: 6 * 60 * 1_000, }); await sandbox.writeFiles([ { - path: '.env', - content: `${agentEnvironment(options.pins)}\n`, + path: `${repoCwd}/.env`, + content: `${agentEnvironment(options.pins)}\n${ + spark + ? [ + `SPARK_API_URL=${sparkConfig.SPARK_API_URL}`, + 'SPARK_PROXY_URL=http://127.0.0.1:40000', + 'SPARK_REDACTION_PATH=/tmp/spark-redactions.json', + ].join('\n') + : '' + }\n`, + mode: 0o600, }, ]); console.log(`${label} run eval`); @@ -449,10 +570,19 @@ async function runPairOnce( '--timeout-sec', String(options.timeoutSec), ], - cwd: sandbox.cwd, + cwd: repoCwd, + // Node reads this startup flag before --env-file is loaded. + env: spark + ? { + HTTPS_PROXY: 'http://127.0.0.1:40000', + NODE_USE_ENV_PROXY: '1', + NO_PROXY: SPARK_PROXY_BYPASS, + DOCKER_BUILDKIT: '0', + } + : undefined, timeoutMs: options.timeoutSec * 1_000 + EVAL_TIMEOUT_BUFFER_MS, }, - true + !spark ); await runSandboxCommand(sandbox, label, 'validate result', { cmd: 'test', @@ -460,32 +590,81 @@ async function runPairOnce( '-f', `results/${pair.eval_suite}/${pair.experiment}/${pair.eval_id}/run-${run}/result.json`, ], - cwd: sandbox.cwd, + cwd: repoCwd, timeoutMs: 30_000, }); - await runSandboxCommand(sandbox, label, 'pack workspace', { - cmd: 'sh', - args: [ - '-c', - packWorkspaceScript( - `results/${pair.eval_suite}/${pair.experiment}/${pair.eval_id}/run-${run}/workspace`, - '/tmp/eval-workspace.tgz' - ), - ], - cwd: sandbox.cwd, - timeoutMs: 3 * 60 * 1_000, - }); + if (spark) { + await runSandboxCommand(sandbox, label, 'sanitize Spark result', { + cmd: 'node', + args: [ + '/tmp/redact-spark-result.ts', + `${repoCwd}/results/${pair.eval_suite}/${pair.experiment}/${pair.eval_id}/run-${run}/result.json`, + ], + sudo: true, + timeoutMs: 30_000, + }); + } else { + await runSandboxCommand(sandbox, label, 'pack workspace', { + cmd: 'sh', + args: [ + '-c', + packWorkspaceScript( + `results/${pair.eval_suite}/${pair.experiment}/${pair.eval_id}/run-${run}/workspace`, + '/tmp/eval-workspace.tgz' + ), + ], + cwd: repoCwd, + timeoutMs: 3 * 60 * 1_000, + }); + } pendingResult = await downloadResults( sandbox, pair, run, - options.outputDir + options.outputDir, + spark ? repoCwd : '' ); } catch (error) { + if (spark) { + const step = + error instanceof SandboxCommandError ? error.step : 'setup or download'; + const safeError = new Error( + `Spark ${step} failed. Inspect private Sandbox logs.` + ); + throw error instanceof AbortError ? new AbortError(safeError) : safeError; + } if (error instanceof Error) throw error; throw new Error(String(error)); } finally { if (sandbox) { + if (sparkSetupInstalled) { + try { + await runSandboxCommand(sandbox, label, 'deregister WARP', { + cmd: 'bash', + args: [ + '-euo', + 'pipefail', + '-c', + `umask 077 +exec >>/tmp/spark-setup.log 2>&1 +rm -f /var/lib/cloudflare-warp/mdm.xml +warp-cli --accept-tos disconnect +warp-cli --accept-tos registration delete +if warp-cli --accept-tos registration show >/dev/null 2>&1; then + echo 'WARP registration still present' >&2 + exit 1 +fi`, + ], + sudo: true, + timeoutMs: 30_000, + }); + console.log(`${label} WARP registration deleted`); + } catch { + console.warn( + `${label} WARP deregistration failed. Continuing VM cleanup.` + ); + } + } const sandboxUsage = await cleanupSandbox(sandbox, label); if (pendingResult) finalizeResult(pendingResult, sandboxUsage); } @@ -664,7 +843,8 @@ export async function downloadResults( sandbox: Pick, pair: EvalPair, run: number, - outputDir: string + outputDir: string, + repoRoot = '' ): Promise { const destination = join( outputDir, @@ -681,27 +861,33 @@ export async function downloadResults( const workspacePartial = `${workspace}.partial`; const resultDownloaded = await sandbox.downloadFile( { - path: `results/${pair.eval_suite}/${pair.experiment}/${pair.eval_id}/run-${run}/result.json`, + path: join( + repoRoot, + `results/${pair.eval_suite}/${pair.experiment}/${pair.eval_id}/run-${run}/result.json` + ), }, { path: resultPartial }, { mkdirRecursive: true } ); if (!resultDownloaded) throw new Error('result.json was missing'); - const workspaceDownloaded = await sandbox.downloadFile( - { path: '/tmp/eval-workspace.tgz' }, - { path: workspacePartial }, - { mkdirRecursive: true } - ); - if (!workspaceDownloaded) throw new Error('workspace archive was missing'); - renameSync(workspacePartial, workspace); - // Missing for agents without a `sessionDir` (e.g. ai-sdk). - await sandbox.downloadFile( - { - path: `results/${pair.eval_suite}/${pair.experiment}/${pair.eval_id}/run-${run}/session-archive.tar.gz`, - }, - { path: join(destination, 'session-archive.tar.gz') }, - { mkdirRecursive: true } - ); + // Spark workspaces and binary sessions can contain project credentials. + if (pair.eval_suite !== 'spark') { + const workspaceDownloaded = await sandbox.downloadFile( + { path: '/tmp/eval-workspace.tgz' }, + { path: workspacePartial }, + { mkdirRecursive: true } + ); + if (!workspaceDownloaded) throw new Error('workspace archive was missing'); + renameSync(workspacePartial, workspace); + // Missing for agents without a `sessionDir` (e.g. ai-sdk). + await sandbox.downloadFile( + { + path: `results/${pair.eval_suite}/${pair.experiment}/${pair.eval_id}/run-${run}/session-archive.tar.gz`, + }, + { path: join(destination, 'session-archive.tar.gz') }, + { mkdirRecursive: true } + ); + } console.log(`${jobLabel(pair, run)} results downloaded to ${destination}`); return { partialPath: resultPartial, finalPath: result }; } diff --git a/apps/framework/scripts/setup-spark-warp.sh b/apps/framework/scripts/setup-spark-warp.sh new file mode 100644 index 00000000..44b0e62f --- /dev/null +++ b/apps/framework/scripts/setup-spark-warp.sh @@ -0,0 +1,31 @@ +#!/bin/bash +umask 077 +exec >>/tmp/spark-setup.log 2>&1 +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y -qq curl gpg ca-certificates docker.io +curl -fsSL https://pkg.cloudflareclient.com/pubkey.gpg | gpg --dearmor --yes -o /usr/share/keyrings/cloudflare-warp.gpg +chmod 644 /usr/share/keyrings/cloudflare-warp.gpg +. /etc/os-release +echo "deb [signed-by=/usr/share/keyrings/cloudflare-warp.gpg] https://pkg.cloudflareclient.com/ $VERSION_CODENAME main" > /etc/apt/sources.list.d/cloudflare-client.list +chmod 644 /etc/apt/sources.list.d/cloudflare-client.list +apt-get update -qq +apt-get install -y -qq cloudflare-warp +update-alternatives --set iptables /usr/sbin/iptables-legacy +update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy +install -D -m 600 /tmp/spark-warp-mdm.xml /var/lib/cloudflare-warp/mdm.xml +# shortcut: Sandbox lacks bridge raw-table support, use host networking for Spark only. +cat > /usr/local/bin/docker <<'DOCKER' +#!/bin/bash +set -euo pipefail +case "${1:-}" in + run|build) + for arg in "$@"; do + case "$arg" in --network|--network=*) exec /usr/bin/docker "$@";; esac + done + exec /usr/bin/docker "$1" --network host "${@:2}" + ;; +esac +exec /usr/bin/docker "$@" +DOCKER +chmod 755 /usr/local/bin/docker diff --git a/experiments/spark/codex-gpt-6-luna.experiment.ts b/experiments/spark/codex-gpt-6-luna.experiment.ts index 3e5c543d..db563416 100644 --- a/experiments/spark/codex-gpt-6-luna.experiment.ts +++ b/experiments/spark/codex-gpt-6-luna.experiment.ts @@ -4,6 +4,10 @@ import { sparkRuntime } from './lib/runtime.js'; export default defineExperiment({ agent: codexGpt6Luna.agent, - runtime: sparkRuntime(process.env.SPARK_API_URL ?? ''), + runtime: sparkRuntime( + process.env.SPARK_API_URL ?? '', + process.env.SPARK_PROXY_URL, + process.env.SPARK_REDACTION_PATH + ), skills: [...codexGpt6Luna.skills, 'spark'], }); diff --git a/experiments/spark/lib/runtime.test.ts b/experiments/spark/lib/runtime.test.ts index d062dc9d..0d368365 100644 --- a/experiments/spark/lib/runtime.test.ts +++ b/experiments/spark/lib/runtime.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, writeFile, rm } from 'node:fs/promises'; +import { mkdtemp, readFile, stat, writeFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, expect, it, vi } from 'vitest'; @@ -43,10 +43,27 @@ it('creates fresh projects, seeds before execution and reads scoring state', asy }); vi.stubGlobal('fetch', fetchMock); try { - const runtime = sparkRuntime('https://control.example'); + const redactionPath = join(dir, 'redactions.json'); + const runtime = sparkRuntime( + 'https://control.example', + 'http://proxy.example', + redactionPath + ); const session = await runtime.startSession({ projectSeedSql: fixture }); + expect(session.mountDockerSocket).toBe(false); expect(fetchMock).toHaveBeenCalledTimes(2); + expect(JSON.parse(await readFile(redactionPath, 'utf8'))).toEqual([ + 'project1', + 'https://project1.example', + 'project1.example', + 'secret', + 'publishable', + ]); + expect((await stat(redactionPath)).mode & 0o777).toBe(0o600); expect(session.env?.SPARK_PROJECT_URL).toBe('https://project1.example'); + expect(session.env?.HTTPS_PROXY).toBe('http://proxy.example'); + expect(session.env?.NO_PROXY).toContain('.openai.com'); + expect(session.env?.NODE_USE_ENV_PROXY).toBe('1'); expect(session.mcpServers).toEqual({}); expect(await session.scoringContext.query('select 1')).toEqual({ rows: [{ value: 1 }], diff --git a/experiments/spark/lib/runtime.ts b/experiments/spark/lib/runtime.ts index 745dd103..ff9713fd 100644 --- a/experiments/spark/lib/runtime.ts +++ b/experiments/spark/lib/runtime.ts @@ -1,4 +1,4 @@ -import { readFile } from 'node:fs/promises'; +import { readFile, writeFile } from 'node:fs/promises'; import { createClient } from '@supabase/supabase-js'; import { createManagementApiClient } from '@supabase-evals/platform-lite'; import { z } from 'zod'; @@ -11,6 +11,9 @@ const projectSchema = z.object({ }); const rowsSchema = z.array(z.record(z.string(), z.unknown())); +export const SPARK_PROXY_BYPASS = + 'localhost,127.0.0.1,.openai.com,.anthropic.com,api.x.ai,ai-gateway.vercel.sh,.npmjs.org,.github.com,.githubusercontent.com'; + /** The Spark project's URL and API keys, like `LocalStackStatus` for the local stack. */ export type SparkProjectStatus = { projectUrl: string; @@ -23,7 +26,11 @@ export type SparkScoringContext = ToolScoringContext & { project: SparkProjectStatus; }; -export function sparkRuntime(apiUrl: string): EvalRuntime { +export function sparkRuntime( + apiUrl: string, + proxyUrl?: string, + redactionPath?: string +): EvalRuntime { return { id: 'spark', async startSession(args) { @@ -47,6 +54,18 @@ export function sparkRuntime(apiUrl: string): EvalRuntime { `Spark project creation failed (${createResponse.status})` ); const project = projectSchema.parse(await createResponse.json()); + if (redactionPath) { + await writeFile( + redactionPath, + JSON.stringify([ + project.ref, + project.url, + new URL(project.url).hostname, + ...project.api_keys.map((key) => key.api_key), + ]), + { mode: 0o600 } + ); + } const secret = project.api_keys.find( (key) => key.type === 'secret' @@ -58,7 +77,7 @@ export function sparkRuntime(apiUrl: string): EvalRuntime { throw new Error(`Spark project ${project.ref} is missing API keys`); // shortcut: projects remain allocated, add teardown once a deletion endpoint is confirmed. - process.stderr.write(`Spark project created: ${project.ref}\n`); + process.stderr.write('Spark project created\n'); const headers = { apikey: secret, 'content-type': 'application/json' }; if (seedSql) { @@ -86,8 +105,17 @@ export function sparkRuntime(apiUrl: string): EvalRuntime { auth: { persistSession: false, autoRefreshToken: false }, }); return { + // WARP credentials live on the VM. + mountDockerSocket: false, mcpServers: {}, env: { + ...(proxyUrl + ? { + HTTPS_PROXY: proxyUrl, + NO_PROXY: SPARK_PROXY_BYPASS, + NODE_USE_ENV_PROXY: '1', + } + : {}), SPARK_PROJECT_URL: project.url, SPARK_SECRET_KEY: secret, SPARK_PUBLISHABLE_KEY: publishable, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 999ffc4a..50e88c36 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -962,6 +962,8 @@ export type EvalSessionArgs = { export type EvalSession = { mcpServers: Record; + /** Whether the agent can control the host Docker daemon. Defaults to true. */ + mountDockerSocket?: boolean; /** Connection settings available to every process in the agent sandbox. */ env?: Record; /** diff --git a/packages/sandbox/src/agent-environment.ts b/packages/sandbox/src/agent-environment.ts index 52b146a7..96c02a72 100644 --- a/packages/sandbox/src/agent-environment.ts +++ b/packages/sandbox/src/agent-environment.ts @@ -34,6 +34,7 @@ export interface LocalStackSetup { } export interface AgentEnvironmentOptions { + mountDockerSocket?: boolean; /** Supabase CLI version baked into the sandbox image. */ cliVersion?: string; /** Host directory whose contents seed the workspace. */ @@ -74,6 +75,7 @@ export async function createAgentEnvironment( const image = await ensureSupabaseSandboxImage(); const sandbox = await DockerSandbox.create({ image, + mountDockerSocket: options.mountDockerSocket, // The local stack needs host networking so the Supabase CLI can health-check // the ports `supabase start` publishes on 127.0.0.1. Tools mode runs no // stack and instead reaches host-side platform-lite over the default bridge diff --git a/packages/sandbox/src/bare-sandbox.ts b/packages/sandbox/src/bare-sandbox.ts index 6b6772d0..97a5a725 100644 --- a/packages/sandbox/src/bare-sandbox.ts +++ b/packages/sandbox/src/bare-sandbox.ts @@ -13,6 +13,7 @@ export interface BareSandboxHandle { export interface BareSandboxOptions { env?: Record; + mountDockerSocket?: boolean; /** Supabase CLI version baked into the sandbox image. */ cliVersion?: string; /** Skills to install into the sandbox. */ @@ -40,6 +41,7 @@ export async function createBareSandbox( ): Promise { const env = await createAgentEnvironment({ env: options.env, + mountDockerSocket: options.mountDockerSocket, cliVersion: options.cliVersion, skills: options.skills, mounts: options.mounts, diff --git a/packages/sandbox/test/docker.test.ts b/packages/sandbox/test/docker.test.ts index ef68bca2..81bbb02d 100644 --- a/packages/sandbox/test/docker.test.ts +++ b/packages/sandbox/test/docker.test.ts @@ -14,6 +14,7 @@ import { localStackRuntime, } from '../src/local-stack-runtime.js'; import { DockerSandbox } from '../src/docker-sandbox.js'; +import { createBareSandbox } from '../src/bare-sandbox.js'; import { ensureSupabaseSandboxImage, setupSupabaseSandbox, @@ -32,6 +33,27 @@ const TEST_TIMEOUT_MS = 600_000; describe.runIf(process.env.SANDBOX_DOCKER_TESTS)( 'local-stack sandbox (docker)', () => { + it( + 'bare agents honor Docker socket access', + { timeout: TEST_TIMEOUT_MS }, + async () => { + for (const mountDockerSocket of [undefined, false]) { + const handle = await createBareSandbox({ mountDockerSocket }); + try { + const result = await handle.sandbox.exec( + 'if [ -S /var/run/docker.sock ]; then printf present; else printf absent; fi' + ); + expect(result.ok).toBe(true); + expect(result.stdout).toBe( + mountDockerSocket === false ? 'absent' : 'present' + ); + } finally { + await handle.close(); + } + } + } + ); + it( 'boots a sandbox, starts a minimal local stack, and reaches it', { timeout: TEST_TIMEOUT_MS },