From 2a93f7c50a12e10720988de8972fea174c882fb1 Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Thu, 12 Nov 2026 14:18:06 -0500
Subject: [PATCH 1/6] feat: let eval runtimes disable Docker socket access

---
 apps/framework/harness/run-eval.ts        |  1 +
 packages/core/src/index.ts                |  2 ++
 packages/sandbox/src/agent-environment.ts |  2 ++
 packages/sandbox/src/bare-sandbox.ts      |  2 ++
 packages/sandbox/test/docker.test.ts      | 22 ++++++++++++++++++++++
 5 files changed, 29 insertions(+)

diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts
index a64926d76..e7917374b 100644
--- a/apps/framework/harness/run-eval.ts
+++ b/apps/framework/harness/run-eval.ts
@@ -530,6 +530,7 @@ async function runOne(
     ? disposable(
         await createBareSandbox({
           env: session.env,
+          mountDockerSocket: session.mountDockerSocket,
           skills: skillSources,
           mounts: supabaseMcpServerMounts(),
         })
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 999ffc4a0..50e88c36e 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -962,6 +962,8 @@ export type EvalSessionArgs = {
 
 export type EvalSession = {
   mcpServers: Record<string, McpServerConfig>;
+  /** Whether the agent can control the host Docker daemon. Defaults to true. */
+  mountDockerSocket?: boolean;
   /** Connection settings available to every process in the agent sandbox. */
   env?: Record<string, string>;
   /**
diff --git a/packages/sandbox/src/agent-environment.ts b/packages/sandbox/src/agent-environment.ts
index 52b146a7a..96c02a72c 100644
--- a/packages/sandbox/src/agent-environment.ts
+++ b/packages/sandbox/src/agent-environment.ts
@@ -34,6 +34,7 @@ export interface LocalStackSetup {
 }
 
 export interface AgentEnvironmentOptions {
+  mountDockerSocket?: boolean;
   /** Supabase CLI version baked into the sandbox image. */
   cliVersion?: string;
   /** Host directory whose contents seed the workspace. */
@@ -74,6 +75,7 @@ export async function createAgentEnvironment(
   const image = await ensureSupabaseSandboxImage();
   const sandbox = await DockerSandbox.create({
     image,
+    mountDockerSocket: options.mountDockerSocket,
     // The local stack needs host networking so the Supabase CLI can health-check
     // the ports `supabase start` publishes on 127.0.0.1. Tools mode runs no
     // stack and instead reaches host-side platform-lite over the default bridge
diff --git a/packages/sandbox/src/bare-sandbox.ts b/packages/sandbox/src/bare-sandbox.ts
index 6b6772d08..97a5a7257 100644
--- a/packages/sandbox/src/bare-sandbox.ts
+++ b/packages/sandbox/src/bare-sandbox.ts
@@ -13,6 +13,7 @@ export interface BareSandboxHandle {
 
 export interface BareSandboxOptions {
   env?: Record<string, string>;
+  mountDockerSocket?: boolean;
   /** Supabase CLI version baked into the sandbox image. */
   cliVersion?: string;
   /** Skills to install into the sandbox. */
@@ -40,6 +41,7 @@ export async function createBareSandbox(
 ): Promise<BareSandboxHandle> {
   const env = await createAgentEnvironment({
     env: options.env,
+    mountDockerSocket: options.mountDockerSocket,
     cliVersion: options.cliVersion,
     skills: options.skills,
     mounts: options.mounts,
diff --git a/packages/sandbox/test/docker.test.ts b/packages/sandbox/test/docker.test.ts
index ef68bca25..81bbb02da 100644
--- a/packages/sandbox/test/docker.test.ts
+++ b/packages/sandbox/test/docker.test.ts
@@ -14,6 +14,7 @@ import {
   localStackRuntime,
 } from '../src/local-stack-runtime.js';
 import { DockerSandbox } from '../src/docker-sandbox.js';
+import { createBareSandbox } from '../src/bare-sandbox.js';
 import {
   ensureSupabaseSandboxImage,
   setupSupabaseSandbox,
@@ -32,6 +33,27 @@ const TEST_TIMEOUT_MS = 600_000;
 describe.runIf(process.env.SANDBOX_DOCKER_TESTS)(
   'local-stack sandbox (docker)',
   () => {
+    it(
+      'bare agents honor Docker socket access',
+      { timeout: TEST_TIMEOUT_MS },
+      async () => {
+        for (const mountDockerSocket of [undefined, false]) {
+          const handle = await createBareSandbox({ mountDockerSocket });
+          try {
+            const result = await handle.sandbox.exec(
+              'if [ -S /var/run/docker.sock ]; then printf present; else printf absent; fi'
+            );
+            expect(result.ok).toBe(true);
+            expect(result.stdout).toBe(
+              mountDockerSocket === false ? 'absent' : 'present'
+            );
+          } finally {
+            await handle.close();
+          }
+        }
+      }
+    );
+
     it(
       'boots a sandbox, starts a minimal local stack, and reaches it',
       { timeout: TEST_TIMEOUT_MS },

From a6843ba10ded2fe144f7edd1ec2615824739724d Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Thu, 12 Nov 2026 14:18:18 -0500
Subject: [PATCH 2/6] feat: enable Spark evals in CI

---
 .github/workflows/eval-refresh.yml            |   3 +
 .github/workflows/test.yml                    |   3 +
 README.md                                     |  25 ++
 .../scripts/run-vercel-evals.test.ts          |  93 ++++++-
 apps/framework/scripts/run-vercel-evals.ts    | 262 ++++++++++++++----
 apps/framework/scripts/setup-spark-warp.py    | 136 +++++++++
 .../spark/codex-gpt-6-luna.experiment.ts      |   6 +-
 experiments/spark/lib/runtime.test.ts         |  21 +-
 experiments/spark/lib/runtime.ts              |  34 ++-
 9 files changed, 515 insertions(+), 68 deletions(-)
 create mode 100644 apps/framework/scripts/setup-spark-warp.py

diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml
index b177f12d9..4bd2d12b4 100644
--- a/.github/workflows/eval-refresh.yml
+++ b/.github/workflows/eval-refresh.yml
@@ -320,6 +320,9 @@ jobs:
       VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
       VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }}
       VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
+      SPARK_API_URL: ${{ secrets.SPARK_API_URL }}
+      CLOUDFLARE_WARP_CLIENT_ID: ${{ secrets.CLOUDFLARE_WARP_CLIENT_ID }}
+      CLOUDFLARE_WARP_CLIENT_SECRET: ${{ secrets.CLOUDFLARE_WARP_CLIENT_SECRET }}
       SUPABASE_CLI_STABLE_VERSION: ${{ needs.prepare.outputs.cli_stable_version }}
       SUPABASE_CLI_BETA_VERSION: ${{ needs.prepare.outputs.cli_beta_version }}
       SUPABASE_CLI_NEXT_VERSION: ${{ needs.prepare.outputs.cli_next_version }}
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5875d0166..a12c3e7bc 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -27,6 +27,9 @@ jobs:
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
+      - name: Check Spark result redaction
+        run: python3 -I apps/framework/scripts/setup-spark-warp.py --self-check
+
       - name: Run tests
         # The framework smoke test calls an LLM judge. This is expected to fail on fork PRs.
         env:
diff --git a/README.md b/README.md
index b859d7ee7..1c22f2d4e 100644
--- a/README.md
+++ b/README.md
@@ -74,6 +74,31 @@ Run every experiment in `experiments/benchmark/` across all benchmark evals:
 pnpm eval -- --suite benchmark
 ```
 
+### Spark in CI
+
+Dispatch the normal workflow with the Spark suite:
+
+```bash
+gh workflow run eval-refresh.yml -R supabase/evals \
+  -f suite=spark -f experiments=codex-gpt-6-luna \
+  -f eval=build-database-001-task-due-dates -f runs=1 \
+  -f commit_results=none
+```
+
+The runner needs `SPARK_API_URL`, `CLOUDFLARE_WARP_CLIENT_ID`, and
+`CLOUDFLARE_WARP_CLIENT_SECRET` in repository secrets, alongside the existing
+Vercel and provider credentials. Only Spark uses the Ubuntu universal image,
+WARP's local HTTP proxy on port 40000, and Docker host networking. Provider
+requests bypass WARP and retain Sandbox credential brokering.
+Spark agents do not receive the host Docker socket. The runner does not
+restrict WARP's proxy to Spark hosts.
+
+Spark results are sanitized inside Sandbox before download. Public artifacts
+contain result JSON without agent workspaces or binary session archives.
+Raw eval output stays inside Sandbox. [Cloudflare documents a 10-second proxy
+request limit](https://developers.cloudflare.com/cloudflare-one/team-and-resources/devices/cloudflare-one-client/configure/modes/#local-proxy-mode), so longer Spark requests may fail. Projects currently remain
+allocated after a run because a deletion endpoint is not implemented.
+
 ### View results in the web app
 
 Set `BRAINTRUST_API_KEY` and `BRAINTRUST_PROJECT_ID` in the repository's `.env`, then run:
diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts
index e7cabc54b..f35e3d9f2 100644
--- a/apps/framework/scripts/run-vercel-evals.test.ts
+++ b/apps/framework/scripts/run-vercel-evals.test.ts
@@ -1,4 +1,5 @@
-import { APIError } from '@vercel/sandbox';
+import { APIError, Sandbox } from '@vercel/sandbox';
+import { AbortError } from 'p-retry';
 import { resolveCliVersion, type CliChannel } from '@supabase-evals/sandbox';
 import { execFileSync } from 'node:child_process';
 import {
@@ -29,6 +30,7 @@ import {
   requiredCliChannels,
   resolveChannelPins,
   runBounded,
+  runPairOnce,
   tagValue,
   expandJobs,
   type EvalPair,
@@ -38,6 +40,61 @@ vi.mock('@supabase-evals/sandbox', () => ({ resolveCliVersion: vi.fn() }));
 
 const BROKERED_NAMES = BROKERED_KEYS.map(({ name }) => name);
 
+describe('Spark terminal failures', () => {
+  const options = {
+    pair: {
+      eval_id: 'eval-1',
+      experiment: 'experiment-1',
+      eval_suite: 'spark',
+    },
+    pairs: [],
+    revision: 'test',
+    repoUrl: 'https://github.com/supabase/evals',
+    outputDir: '/tmp/unused-spark-results',
+    runs: 1,
+    timeoutSec: 1,
+    concurrency: 1,
+    vcpus: 1,
+    attempt: 1,
+    run: 1,
+    pins: {},
+  };
+  const credentials = { token: 'test', teamId: 'test', projectId: 'test' };
+
+  beforeEach(() => {
+    for (const name of BROKERED_NAMES) vi.stubEnv(name, '');
+    vi.stubEnv('SPARK_API_URL', 'https://control.example');
+    vi.stubEnv('CLOUDFLARE_WARP_CLIENT_ID', 'test.access');
+    vi.stubEnv('CLOUDFLARE_WARP_CLIENT_SECRET', 'test');
+    vi.spyOn(console, 'warn').mockImplementation(() => {});
+  });
+  afterEach(() => {
+    vi.unstubAllEnvs();
+    vi.restoreAllMocks();
+  });
+
+  it('preserves terminal create failures so the outer retry loop stops', async () => {
+    const create = vi
+      .spyOn(Sandbox, 'create')
+      .mockRejectedValue(new APIError(new Response(null, { status: 401 })));
+    await expect(runPairOnce(options, credentials)).rejects.toBeInstanceOf(
+      AbortError
+    );
+    expect(create).toHaveBeenCalledTimes(1);
+  });
+
+  it('does not retry missing credentials or allocate a VM', async () => {
+    vi.stubEnv('CLOUDFLARE_WARP_CLIENT_SECRET', '');
+    const create = vi
+      .spyOn(Sandbox, 'create')
+      .mockRejectedValue(new APIError(new Response(null, { status: 401 })));
+    await expect(runPairOnce(options, credentials)).rejects.toBeInstanceOf(
+      AbortError
+    );
+    expect(create).not.toHaveBeenCalled();
+  });
+});
+
 describe('agentEnvironment', () => {
   const originalValues = new Map<string, string | undefined>();
 
@@ -422,6 +479,40 @@ describe('Vercel eval controller', () => {
     }
   });
 
+  it('downloads only sanitized JSON for Spark, never workspace or session archives', async () => {
+    const temporary = mkdtempSync(join(tmpdir(), 'spark-download-test-'));
+    const paths: string[] = [];
+    try {
+      await downloadResults(
+        {
+          downloadFile: async (source, destination) => {
+            paths.push(source.path);
+            mkdirSync(dirname(destination.path), { recursive: true });
+            writeFileSync(destination.path, '{"passed":true}');
+            return destination.path;
+          },
+        },
+        { eval_id: 'eval-1', experiment: 'experiment-1', eval_suite: 'spark' },
+        1,
+        temporary,
+        '/vercel/evals'
+      );
+      expect(paths).toEqual([
+        '/vercel/evals/results/spark/experiment-1/eval-1/run-1/result.json',
+      ]);
+      expect(
+        readdirSync(
+          join(
+            temporary,
+            'raw-results-spark__experiment-1__eval-1/eval-1/run-1'
+          )
+        )
+      ).toEqual(['result.json.partial']);
+    } finally {
+      rmSync(temporary, { recursive: true, force: true });
+    }
+  });
+
   it('downloads result metadata separately from agent workspace files', async () => {
     const temporary = mkdtempSync(join(tmpdir(), 'vercel-eval-test-'));
     const sandboxFiles = join(temporary, 'sandbox');
diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts
index aaf7318fb..5fe34e717 100644
--- a/apps/framework/scripts/run-vercel-evals.ts
+++ b/apps/framework/scripts/run-vercel-evals.ts
@@ -22,6 +22,7 @@ import pLimit from 'p-limit';
 import pRetry, { AbortError } from 'p-retry';
 import { z } from 'zod';
 import { positiveInteger, readFlag } from '../lib/cli-args.js';
+import { SPARK_PROXY_BYPASS } from '../../../experiments/spark/lib/runtime.js';
 import { discoverExperimentFiles } from '../lib/experiment-files.js';
 
 const ROOT = fileURLToPath(new URL('../../../', import.meta.url));
@@ -234,7 +235,11 @@ interface SandboxCommandOptions {
 }
 
 class SandboxCommandError extends Error {
-  constructor(step: string, exitCode: number, output: string) {
+  constructor(
+    readonly step: string,
+    exitCode: number,
+    output: string
+  ) {
     const detail = output.trim().slice(-4_000);
     super(`${step} exited with code ${exitCode}${detail ? `\n${detail}` : ''}`);
     this.name = 'SandboxCommandError';
@@ -333,7 +338,7 @@ async function runPairs(options: RunnerOptions): Promise<void> {
   }
 }
 
-async function runPairOnce(
+export async function runPairOnce(
   options: PairOptions,
   credentials: VercelCredentials
 ): Promise<void> {
@@ -341,12 +346,29 @@ async function runPairOnce(
   const label = jobLabel(pair, run);
   let sandbox: Sandbox | undefined;
   let pendingResult: PendingResult | undefined;
+  let sparkSetupInstalled = false;
+  const spark = pair.eval_suite === 'spark';
+  const sparkConfig = spark
+    ? Object.fromEntries(
+        [
+          'CLOUDFLARE_WARP_CLIENT_ID',
+          'CLOUDFLARE_WARP_CLIENT_SECRET',
+          'SPARK_API_URL',
+        ].map((name) => {
+          const value = process.env[name];
+          if (!value) throw new AbortError(`Missing ${name}`);
+          return [name, value];
+        })
+      )
+    : {};
 
   try {
     sandbox = await createSandbox(label, {
       ...credentials,
       name: sandboxName(pair, run),
-      runtime: 'node24',
+      ...(spark
+        ? { image: 'vercel/sandbox/universal' }
+        : { runtime: 'node24' }),
       source: {
         type: 'git',
         url: options.repoUrl,
@@ -372,6 +394,7 @@ async function runPairOnce(
       `${label} attempt ${options.attempt}: ${SANDBOX_DASHBOARD_URL}/${sandbox.name}`
     );
 
+    const repoCwd = spark ? `${sandbox.cwd}/evals` : sandbox.cwd;
     await runSandboxCommand(sandbox, label, 'initialize submodules', {
       cmd: 'git',
       args: [
@@ -382,29 +405,73 @@ async function runPairOnce(
         '--init',
         '--recursive',
       ],
-      cwd: sandbox.cwd,
+      cwd: repoCwd,
       timeoutMs: 2 * 60 * 1_000,
     });
-    await runSandboxCommand(sandbox, label, 'install Docker', {
-      cmd: 'dnf',
-      args: ['install', '-y', '-q', 'docker'],
-      sudo: true,
-      timeoutMs: 3 * 60 * 1_000,
-    });
-    await sandbox.runCommand({
-      cmd: 'dockerd',
-      sudo: true,
-      detached: true,
-    });
-    await runSandboxCommand(sandbox, label, 'start Docker', {
-      cmd: 'bash',
-      args: [
-        '-c',
-        'for i in $(seq 60); do docker info >/dev/null 2>&1 && chmod 666 /var/run/docker.sock && exit 0; sleep 1; done; echo "dockerd not ready" >&2; exit 1',
-      ],
-      sudo: true,
-      timeoutMs: 90_000,
-    });
+    if (spark) {
+      await sandbox.writeFiles([
+        {
+          path: '/tmp/setup-spark-warp.py',
+          content: readFileSync(
+            new URL('./setup-spark-warp.py', import.meta.url)
+          ),
+        },
+        {
+          path: '/tmp/spark-warp-config.json',
+          content: Buffer.from(JSON.stringify(sparkConfig)),
+          mode: 0o600,
+        },
+      ]);
+      await runSandboxCommand(
+        sandbox,
+        label,
+        'install Spark WARP proxy',
+        {
+          cmd: 'python3',
+          args: ['/tmp/setup-spark-warp.py'],
+          sudo: true,
+          timeoutMs: 10 * 60 * 1_000,
+        },
+        true
+      );
+      sparkSetupInstalled = true;
+      for (const cmd of ['warp-svc', 'dockerd']) {
+        await sandbox.runCommand({ cmd, sudo: true, detached: true });
+      }
+      await runSandboxCommand(
+        sandbox,
+        label,
+        'connect Spark WARP proxy',
+        {
+          cmd: 'python3',
+          args: ['/tmp/setup-spark-warp.py', 'connect'],
+          sudo: true,
+          timeoutMs: 3 * 60 * 1_000,
+        },
+        true
+      );
+    } else {
+      await runSandboxCommand(sandbox, label, 'install Docker', {
+        cmd: 'dnf',
+        args: ['install', '-y', '-q', 'docker'],
+        sudo: true,
+        timeoutMs: 3 * 60 * 1_000,
+      });
+      await sandbox.runCommand({
+        cmd: 'dockerd',
+        sudo: true,
+        detached: true,
+      });
+      await runSandboxCommand(sandbox, label, 'start Docker', {
+        cmd: 'bash',
+        args: [
+          '-c',
+          'for i in $(seq 60); do docker info >/dev/null 2>&1 && chmod 666 /var/run/docker.sock && exit 0; sleep 1; done; echo "dockerd not ready" >&2; exit 1',
+        ],
+        sudo: true,
+        timeoutMs: 90_000,
+      });
+    }
     // Installs the pnpm version pinned by packageManager in the checked-out
     // root package.json so the two can't drift.
     await runSandboxCommand(sandbox, label, 'install pnpm', {
@@ -413,21 +480,30 @@ async function runPairOnce(
         '-c',
         `npm install --global "$(node -p 'require("./package.json").packageManager')"`,
       ],
-      cwd: sandbox.cwd,
+      cwd: repoCwd,
       sudo: true,
       timeoutMs: 2 * 60 * 1_000,
     });
     await runSandboxCommand(sandbox, label, 'install dependencies', {
       cmd: 'pnpm',
       args: ['install', '--frozen-lockfile'],
-      cwd: sandbox.cwd,
+      cwd: repoCwd,
       timeoutMs: 6 * 60 * 1_000,
     });
 
     await sandbox.writeFiles([
       {
-        path: '.env',
-        content: `${agentEnvironment(options.pins)}\n`,
+        path: `${repoCwd}/.env`,
+        content: `${agentEnvironment(options.pins)}\n${
+          spark
+            ? [
+                `SPARK_API_URL=${sparkConfig.SPARK_API_URL}`,
+                'SPARK_PROXY_URL=http://127.0.0.1:40000',
+                'SPARK_REDACTION_PATH=/tmp/spark-redactions.json',
+              ].join('\n')
+            : ''
+        }\n`,
+        mode: 0o600,
       },
     ]);
     console.log(`${label} run eval`);
@@ -449,10 +525,19 @@ async function runPairOnce(
           '--timeout-sec',
           String(options.timeoutSec),
         ],
-        cwd: sandbox.cwd,
+        cwd: repoCwd,
+        // Node reads this startup flag before --env-file is loaded.
+        env: spark
+          ? {
+              HTTPS_PROXY: 'http://127.0.0.1:40000',
+              NODE_USE_ENV_PROXY: '1',
+              NO_PROXY: SPARK_PROXY_BYPASS,
+              DOCKER_BUILDKIT: '0',
+            }
+          : undefined,
         timeoutMs: options.timeoutSec * 1_000 + EVAL_TIMEOUT_BUFFER_MS,
       },
-      true
+      !spark
     );
     await runSandboxCommand(sandbox, label, 'validate result', {
       cmd: 'test',
@@ -460,32 +545,80 @@ async function runPairOnce(
         '-f',
         `results/${pair.eval_suite}/${pair.experiment}/${pair.eval_id}/run-${run}/result.json`,
       ],
-      cwd: sandbox.cwd,
+      cwd: repoCwd,
       timeoutMs: 30_000,
     });
-    await runSandboxCommand(sandbox, label, 'pack workspace', {
-      cmd: 'sh',
-      args: [
-        '-c',
-        packWorkspaceScript(
-          `results/${pair.eval_suite}/${pair.experiment}/${pair.eval_id}/run-${run}/workspace`,
-          '/tmp/eval-workspace.tgz'
-        ),
-      ],
-      cwd: sandbox.cwd,
-      timeoutMs: 3 * 60 * 1_000,
-    });
+    if (spark) {
+      await runSandboxCommand(sandbox, label, 'sanitize Spark result', {
+        cmd: 'python3',
+        args: [
+          '/tmp/setup-spark-warp.py',
+          'sanitize',
+          `${repoCwd}/results/${pair.eval_suite}/${pair.experiment}/${pair.eval_id}/run-${run}/result.json`,
+        ],
+        sudo: true,
+        timeoutMs: 30_000,
+      });
+    } else {
+      await runSandboxCommand(sandbox, label, 'pack workspace', {
+        cmd: 'sh',
+        args: [
+          '-c',
+          packWorkspaceScript(
+            `results/${pair.eval_suite}/${pair.experiment}/${pair.eval_id}/run-${run}/workspace`,
+            '/tmp/eval-workspace.tgz'
+          ),
+        ],
+        cwd: repoCwd,
+        timeoutMs: 3 * 60 * 1_000,
+      });
+    }
     pendingResult = await downloadResults(
       sandbox,
       pair,
       run,
-      options.outputDir
+      options.outputDir,
+      spark ? repoCwd : ''
     );
   } catch (error) {
+    if (spark) {
+      const step =
+        error instanceof SandboxCommandError ? error.step : 'setup or download';
+      const receipt =
+        error instanceof Error
+          ? error.message
+              .match(/Spark setup error type [A-Za-z]+ at line [0-9]+/g)
+              ?.join(', ')
+          : undefined;
+      const safeError = new Error(
+        `Spark ${step} failed. ${receipt ?? 'Inspect private Sandbox logs.'}`
+      );
+      throw error instanceof AbortError ? new AbortError(safeError) : safeError;
+    }
     if (error instanceof Error) throw error;
     throw new Error(String(error));
   } finally {
     if (sandbox) {
+      if (sparkSetupInstalled) {
+        try {
+          await runSandboxCommand(
+            sandbox,
+            label,
+            'deregister WARP',
+            {
+              cmd: 'python3',
+              args: ['/tmp/setup-spark-warp.py', 'cleanup'],
+              sudo: true,
+              timeoutMs: 30_000,
+            },
+            true
+          );
+        } catch {
+          console.warn(
+            `${label} WARP deregistration failed. Continuing VM cleanup.`
+          );
+        }
+      }
       const sandboxUsage = await cleanupSandbox(sandbox, label);
       if (pendingResult) finalizeResult(pendingResult, sandboxUsage);
     }
@@ -664,7 +797,8 @@ export async function downloadResults(
   sandbox: Pick<Sandbox, 'downloadFile'>,
   pair: EvalPair,
   run: number,
-  outputDir: string
+  outputDir: string,
+  repoRoot = ''
 ): Promise<PendingResult> {
   const destination = join(
     outputDir,
@@ -681,27 +815,33 @@ export async function downloadResults(
   const workspacePartial = `${workspace}.partial`;
   const resultDownloaded = await sandbox.downloadFile(
     {
-      path: `results/${pair.eval_suite}/${pair.experiment}/${pair.eval_id}/run-${run}/result.json`,
+      path: join(
+        repoRoot,
+        `results/${pair.eval_suite}/${pair.experiment}/${pair.eval_id}/run-${run}/result.json`
+      ),
     },
     { path: resultPartial },
     { mkdirRecursive: true }
   );
   if (!resultDownloaded) throw new Error('result.json was missing');
-  const workspaceDownloaded = await sandbox.downloadFile(
-    { path: '/tmp/eval-workspace.tgz' },
-    { path: workspacePartial },
-    { mkdirRecursive: true }
-  );
-  if (!workspaceDownloaded) throw new Error('workspace archive was missing');
-  renameSync(workspacePartial, workspace);
-  // Missing for agents without a `sessionDir` (e.g. ai-sdk).
-  await sandbox.downloadFile(
-    {
-      path: `results/${pair.eval_suite}/${pair.experiment}/${pair.eval_id}/run-${run}/session-archive.tar.gz`,
-    },
-    { path: join(destination, 'session-archive.tar.gz') },
-    { mkdirRecursive: true }
-  );
+  // Spark workspaces and binary sessions can contain project credentials.
+  if (pair.eval_suite !== 'spark') {
+    const workspaceDownloaded = await sandbox.downloadFile(
+      { path: '/tmp/eval-workspace.tgz' },
+      { path: workspacePartial },
+      { mkdirRecursive: true }
+    );
+    if (!workspaceDownloaded) throw new Error('workspace archive was missing');
+    renameSync(workspacePartial, workspace);
+    // Missing for agents without a `sessionDir` (e.g. ai-sdk).
+    await sandbox.downloadFile(
+      {
+        path: `results/${pair.eval_suite}/${pair.experiment}/${pair.eval_id}/run-${run}/session-archive.tar.gz`,
+      },
+      { path: join(destination, 'session-archive.tar.gz') },
+      { mkdirRecursive: true }
+    );
+  }
   console.log(`${jobLabel(pair, run)} results downloaded to ${destination}`);
   return { partialPath: resultPartial, finalPath: result };
 }
diff --git a/apps/framework/scripts/setup-spark-warp.py b/apps/framework/scripts/setup-spark-warp.py
new file mode 100644
index 000000000..8d34bc350
--- /dev/null
+++ b/apps/framework/scripts/setup-spark-warp.py
@@ -0,0 +1,136 @@
+"""Spark-only Sandbox bootstrap. Raw command output stays inside the VM."""
+
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import time
+import traceback
+from urllib.parse import quote, urlsplit
+from xml.etree.ElementTree import Element, SubElement, tostring
+
+
+def checked(*args):
+    result = subprocess.run(args, capture_output=True, text=True, timeout=300)
+    if result.returncode:
+        Path("/tmp/spark-setup-error.log").write_text(result.stdout + result.stderr)
+        print(f"Spark setup executable {args[0]} failed with exit {result.returncode}", file=sys.stderr)
+        raise RuntimeError("Spark setup command failed")
+    return result.stdout
+
+
+def redact(value, secrets):
+    if isinstance(value, str):
+        for secret in sorted(set(secrets), key=len, reverse=True):
+            if secret:
+                value = value.replace(secret, "[REDACTED]")
+                value = value.replace(quote(secret, safe=""), "[REDACTED]")
+        return value
+    if isinstance(value, list):
+        return [redact(item, secrets) for item in value]
+    if isinstance(value, dict):
+        return {redact(key, secrets): redact(item, secrets) for key, item in value.items()}
+    return value
+
+
+def sanitize(path):
+    config = json.loads(Path("/tmp/spark-warp-config.json").read_text())
+    secrets = list(config.values())
+    hostname = urlsplit(config["SPARK_API_URL"]).hostname
+    if hostname:
+        secrets.append(hostname)
+    secrets.extend(json.loads(Path("/tmp/spark-redactions.json").read_text()))
+    result = Path(path)
+    payload = json.dumps(redact(json.loads(result.read_text()), secrets))
+    if any(secret and (secret in payload or quote(secret, safe="") in payload) for secret in secrets):
+        raise RuntimeError("Result still contains a private value")
+    result.write_text(payload)
+
+
+def setup():
+    os.umask(0o077)
+    config = json.loads(Path("/tmp/spark-warp-config.json").read_text())
+    # shortcut: Sandbox lacks bridge raw-table support, use host networking for Spark only.
+    wrapper = Path("/usr/local/bin/docker")
+    wrapper.write_text('''#!/usr/bin/env python3
+import os, sys
+args = sys.argv[1:]
+if args and args[0] in ('run', 'build') and '--network' not in args:
+    args[1:1] = ['--network', 'host']
+os.execv('/usr/bin/docker', ['docker', *args])
+''')
+    wrapper.chmod(0o755)
+    checked("bash", "-c", """set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+apt-get update -qq
+apt-get install -y -qq curl gpg ca-certificates docker.io
+curl -fsSL https://pkg.cloudflareclient.com/pubkey.gpg | gpg --dearmor --yes -o /usr/share/keyrings/cloudflare-warp.gpg
+chmod 644 /usr/share/keyrings/cloudflare-warp.gpg
+. /etc/os-release
+echo "deb [signed-by=/usr/share/keyrings/cloudflare-warp.gpg] https://pkg.cloudflareclient.com/ $VERSION_CODENAME main" > /etc/apt/sources.list.d/cloudflare-client.list
+chmod 644 /etc/apt/sources.list.d/cloudflare-client.list
+apt-get update -qq
+apt-get install -y -qq cloudflare-warp
+update-alternatives --set iptables /usr/sbin/iptables-legacy
+update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
+""")
+    document = Element("dict")
+    for key, value in [("organization", "supabase"), ("auth_client_id", config["CLOUDFLARE_WARP_CLIENT_ID"]), ("auth_client_secret", config["CLOUDFLARE_WARP_CLIENT_SECRET"]), ("service_mode", "proxy"), ("warp_tunnel_protocol", "masque")]:
+        SubElement(document, "key").text = key
+        SubElement(document, "string").text = value
+    SubElement(document, "key").text = "proxy_port"
+    SubElement(document, "integer").text = "40000"
+    mdm = Path("/var/lib/cloudflare-warp/mdm.xml")
+    mdm.parent.mkdir(parents=True, exist_ok=True)
+    mdm.write_bytes(tostring(document))
+    mdm.chmod(0o600)
+
+
+def connect():
+    for _ in range(60):
+        registration = subprocess.run(["warp-cli", "--accept-tos", "registration", "show"], capture_output=True)
+        if registration.returncode == 0:
+            break
+        time.sleep(1)
+    checked("warp-cli", "--accept-tos", "connect")
+    for _ in range(60):
+        warp = subprocess.run(["warp-cli", "--accept-tos", "status"], capture_output=True, text=True)
+        docker = subprocess.run(["docker", "info"], capture_output=True)
+        if "Status update: Connected" in warp.stdout and docker.returncode == 0:
+            Path("/var/run/docker.sock").chmod(0o666)
+            return
+        time.sleep(1)
+    raise RuntimeError("WARP did not connect")
+
+
+def cleanup():
+    Path("/var/lib/cloudflare-warp/mdm.xml").unlink(missing_ok=True)
+    checked("warp-cli", "--accept-tos", "disconnect")
+    checked("warp-cli", "--accept-tos", "registration", "delete")
+    registration = subprocess.run(["warp-cli", "--accept-tos", "registration", "show"], capture_output=True)
+    if registration.returncode == 0:
+        raise RuntimeError("WARP registration still present")
+    print("WARP registration deleted")
+
+
+if __name__ == "__main__":
+    try:
+        if sys.argv[1:] == ["--self-check"]:
+            secret = "https://private.example/a?token=secret"
+            assert redact({"tool": "warp-client-secret", "project": "spark-project-key"}, ["warp-client-secret", "spark-project-key"]) == {"tool": "[REDACTED]", "project": "[REDACTED]"}
+            assert redact({"nested": [secret, quote(secret, safe=""), 4]}, [secret]) == {"nested": ["[REDACTED]", "[REDACTED]", 4]}
+        elif sys.argv[1:2] == ["sanitize"] and len(sys.argv) == 3:
+            sanitize(sys.argv[2])
+        elif sys.argv[1:] == ["cleanup"]:
+            cleanup()
+        elif sys.argv[1:] == ["connect"]:
+            connect()
+        elif not sys.argv[1:]:
+            setup()
+        else:
+            raise RuntimeError("Invalid arguments")
+    except Exception as error:
+        print(f"Spark setup error type {type(error).__name__} at line {traceback.extract_tb(error.__traceback__)[-1].lineno}", file=sys.stderr)
+        print("Spark setup or result sanitization failed. Inspect private Sandbox logs.", file=sys.stderr)
+        sys.exit(1)
diff --git a/experiments/spark/codex-gpt-6-luna.experiment.ts b/experiments/spark/codex-gpt-6-luna.experiment.ts
index 3e5c543d3..db5634160 100644
--- a/experiments/spark/codex-gpt-6-luna.experiment.ts
+++ b/experiments/spark/codex-gpt-6-luna.experiment.ts
@@ -4,6 +4,10 @@ import { sparkRuntime } from './lib/runtime.js';
 
 export default defineExperiment({
   agent: codexGpt6Luna.agent,
-  runtime: sparkRuntime(process.env.SPARK_API_URL ?? ''),
+  runtime: sparkRuntime(
+    process.env.SPARK_API_URL ?? '',
+    process.env.SPARK_PROXY_URL,
+    process.env.SPARK_REDACTION_PATH
+  ),
   skills: [...codexGpt6Luna.skills, 'spark'],
 });
diff --git a/experiments/spark/lib/runtime.test.ts b/experiments/spark/lib/runtime.test.ts
index d062dc9d1..0d3683650 100644
--- a/experiments/spark/lib/runtime.test.ts
+++ b/experiments/spark/lib/runtime.test.ts
@@ -1,4 +1,4 @@
-import { mkdtemp, writeFile, rm } from 'node:fs/promises';
+import { mkdtemp, readFile, stat, writeFile, rm } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { afterEach, expect, it, vi } from 'vitest';
@@ -43,10 +43,27 @@ it('creates fresh projects, seeds before execution and reads scoring state', asy
   });
   vi.stubGlobal('fetch', fetchMock);
   try {
-    const runtime = sparkRuntime('https://control.example');
+    const redactionPath = join(dir, 'redactions.json');
+    const runtime = sparkRuntime(
+      'https://control.example',
+      'http://proxy.example',
+      redactionPath
+    );
     const session = await runtime.startSession({ projectSeedSql: fixture });
+    expect(session.mountDockerSocket).toBe(false);
     expect(fetchMock).toHaveBeenCalledTimes(2);
+    expect(JSON.parse(await readFile(redactionPath, 'utf8'))).toEqual([
+      'project1',
+      'https://project1.example',
+      'project1.example',
+      'secret',
+      'publishable',
+    ]);
+    expect((await stat(redactionPath)).mode & 0o777).toBe(0o600);
     expect(session.env?.SPARK_PROJECT_URL).toBe('https://project1.example');
+    expect(session.env?.HTTPS_PROXY).toBe('http://proxy.example');
+    expect(session.env?.NO_PROXY).toContain('.openai.com');
+    expect(session.env?.NODE_USE_ENV_PROXY).toBe('1');
     expect(session.mcpServers).toEqual({});
     expect(await session.scoringContext.query('select 1')).toEqual({
       rows: [{ value: 1 }],
diff --git a/experiments/spark/lib/runtime.ts b/experiments/spark/lib/runtime.ts
index 745dd1032..ff9713fdd 100644
--- a/experiments/spark/lib/runtime.ts
+++ b/experiments/spark/lib/runtime.ts
@@ -1,4 +1,4 @@
-import { readFile } from 'node:fs/promises';
+import { readFile, writeFile } from 'node:fs/promises';
 import { createClient } from '@supabase/supabase-js';
 import { createManagementApiClient } from '@supabase-evals/platform-lite';
 import { z } from 'zod';
@@ -11,6 +11,9 @@ const projectSchema = z.object({
 });
 const rowsSchema = z.array(z.record(z.string(), z.unknown()));
 
+export const SPARK_PROXY_BYPASS =
+  'localhost,127.0.0.1,.openai.com,.anthropic.com,api.x.ai,ai-gateway.vercel.sh,.npmjs.org,.github.com,.githubusercontent.com';
+
 /** The Spark project's URL and API keys, like `LocalStackStatus` for the local stack. */
 export type SparkProjectStatus = {
   projectUrl: string;
@@ -23,7 +26,11 @@ export type SparkScoringContext = ToolScoringContext & {
   project: SparkProjectStatus;
 };
 
-export function sparkRuntime(apiUrl: string): EvalRuntime {
+export function sparkRuntime(
+  apiUrl: string,
+  proxyUrl?: string,
+  redactionPath?: string
+): EvalRuntime {
   return {
     id: 'spark',
     async startSession(args) {
@@ -47,6 +54,18 @@ export function sparkRuntime(apiUrl: string): EvalRuntime {
           `Spark project creation failed (${createResponse.status})`
         );
       const project = projectSchema.parse(await createResponse.json());
+      if (redactionPath) {
+        await writeFile(
+          redactionPath,
+          JSON.stringify([
+            project.ref,
+            project.url,
+            new URL(project.url).hostname,
+            ...project.api_keys.map((key) => key.api_key),
+          ]),
+          { mode: 0o600 }
+        );
+      }
 
       const secret = project.api_keys.find(
         (key) => key.type === 'secret'
@@ -58,7 +77,7 @@ export function sparkRuntime(apiUrl: string): EvalRuntime {
         throw new Error(`Spark project ${project.ref} is missing API keys`);
 
       // shortcut: projects remain allocated, add teardown once a deletion endpoint is confirmed.
-      process.stderr.write(`Spark project created: ${project.ref}\n`);
+      process.stderr.write('Spark project created\n');
 
       const headers = { apikey: secret, 'content-type': 'application/json' };
       if (seedSql) {
@@ -86,8 +105,17 @@ export function sparkRuntime(apiUrl: string): EvalRuntime {
           auth: { persistSession: false, autoRefreshToken: false },
         });
       return {
+        // WARP credentials live on the VM.
+        mountDockerSocket: false,
         mcpServers: {},
         env: {
+          ...(proxyUrl
+            ? {
+                HTTPS_PROXY: proxyUrl,
+                NO_PROXY: SPARK_PROXY_BYPASS,
+                NODE_USE_ENV_PROXY: '1',
+              }
+            : {}),
           SPARK_PROJECT_URL: project.url,
           SPARK_SECRET_KEY: secret,
           SPARK_PUBLISHABLE_KEY: publishable,

From 97193046c1dd98b4cce465e2f4e169dea34ad5e8 Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Thu, 12 Nov 2026 14:46:24 -0500
Subject: [PATCH 3/6] fix: use TypeScript for Spark setup and token redaction

---
 .github/workflows/test.yml                    |   3 -
 apps/framework/scripts/run-vercel-evals.ts    |  20 +-
 apps/framework/scripts/setup-spark-warp.py    | 136 ---------
 .../scripts/setup-spark-warp.test.ts          |  56 ++++
 apps/framework/scripts/setup-spark-warp.ts    | 264 ++++++++++++++++++
 5 files changed, 330 insertions(+), 149 deletions(-)
 delete mode 100644 apps/framework/scripts/setup-spark-warp.py
 create mode 100644 apps/framework/scripts/setup-spark-warp.test.ts
 create mode 100644 apps/framework/scripts/setup-spark-warp.ts

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a12c3e7bc..5875d0166 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -27,9 +27,6 @@ jobs:
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
-      - name: Check Spark result redaction
-        run: python3 -I apps/framework/scripts/setup-spark-warp.py --self-check
-
       - name: Run tests
         # The framework smoke test calls an LLM judge. This is expected to fail on fork PRs.
         env:
diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts
index 5fe34e717..4434719d1 100644
--- a/apps/framework/scripts/run-vercel-evals.ts
+++ b/apps/framework/scripts/run-vercel-evals.ts
@@ -411,9 +411,9 @@ export async function runPairOnce(
     if (spark) {
       await sandbox.writeFiles([
         {
-          path: '/tmp/setup-spark-warp.py',
+          path: '/tmp/setup-spark-warp.ts',
           content: readFileSync(
-            new URL('./setup-spark-warp.py', import.meta.url)
+            new URL('./setup-spark-warp.ts', import.meta.url)
           ),
         },
         {
@@ -427,8 +427,8 @@ export async function runPairOnce(
         label,
         'install Spark WARP proxy',
         {
-          cmd: 'python3',
-          args: ['/tmp/setup-spark-warp.py'],
+          cmd: 'node',
+          args: ['/tmp/setup-spark-warp.ts'],
           sudo: true,
           timeoutMs: 10 * 60 * 1_000,
         },
@@ -443,8 +443,8 @@ export async function runPairOnce(
         label,
         'connect Spark WARP proxy',
         {
-          cmd: 'python3',
-          args: ['/tmp/setup-spark-warp.py', 'connect'],
+          cmd: 'node',
+          args: ['/tmp/setup-spark-warp.ts', 'connect'],
           sudo: true,
           timeoutMs: 3 * 60 * 1_000,
         },
@@ -550,9 +550,9 @@ export async function runPairOnce(
     });
     if (spark) {
       await runSandboxCommand(sandbox, label, 'sanitize Spark result', {
-        cmd: 'python3',
+        cmd: 'node',
         args: [
-          '/tmp/setup-spark-warp.py',
+          '/tmp/setup-spark-warp.ts',
           'sanitize',
           `${repoCwd}/results/${pair.eval_suite}/${pair.experiment}/${pair.eval_id}/run-${run}/result.json`,
         ],
@@ -606,8 +606,8 @@ export async function runPairOnce(
             label,
             'deregister WARP',
             {
-              cmd: 'python3',
-              args: ['/tmp/setup-spark-warp.py', 'cleanup'],
+              cmd: 'node',
+              args: ['/tmp/setup-spark-warp.ts', 'cleanup'],
               sudo: true,
               timeoutMs: 30_000,
             },
diff --git a/apps/framework/scripts/setup-spark-warp.py b/apps/framework/scripts/setup-spark-warp.py
deleted file mode 100644
index 8d34bc350..000000000
--- a/apps/framework/scripts/setup-spark-warp.py
+++ /dev/null
@@ -1,136 +0,0 @@
-"""Spark-only Sandbox bootstrap. Raw command output stays inside the VM."""
-
-import json
-import os
-from pathlib import Path
-import subprocess
-import sys
-import time
-import traceback
-from urllib.parse import quote, urlsplit
-from xml.etree.ElementTree import Element, SubElement, tostring
-
-
-def checked(*args):
-    result = subprocess.run(args, capture_output=True, text=True, timeout=300)
-    if result.returncode:
-        Path("/tmp/spark-setup-error.log").write_text(result.stdout + result.stderr)
-        print(f"Spark setup executable {args[0]} failed with exit {result.returncode}", file=sys.stderr)
-        raise RuntimeError("Spark setup command failed")
-    return result.stdout
-
-
-def redact(value, secrets):
-    if isinstance(value, str):
-        for secret in sorted(set(secrets), key=len, reverse=True):
-            if secret:
-                value = value.replace(secret, "[REDACTED]")
-                value = value.replace(quote(secret, safe=""), "[REDACTED]")
-        return value
-    if isinstance(value, list):
-        return [redact(item, secrets) for item in value]
-    if isinstance(value, dict):
-        return {redact(key, secrets): redact(item, secrets) for key, item in value.items()}
-    return value
-
-
-def sanitize(path):
-    config = json.loads(Path("/tmp/spark-warp-config.json").read_text())
-    secrets = list(config.values())
-    hostname = urlsplit(config["SPARK_API_URL"]).hostname
-    if hostname:
-        secrets.append(hostname)
-    secrets.extend(json.loads(Path("/tmp/spark-redactions.json").read_text()))
-    result = Path(path)
-    payload = json.dumps(redact(json.loads(result.read_text()), secrets))
-    if any(secret and (secret in payload or quote(secret, safe="") in payload) for secret in secrets):
-        raise RuntimeError("Result still contains a private value")
-    result.write_text(payload)
-
-
-def setup():
-    os.umask(0o077)
-    config = json.loads(Path("/tmp/spark-warp-config.json").read_text())
-    # shortcut: Sandbox lacks bridge raw-table support, use host networking for Spark only.
-    wrapper = Path("/usr/local/bin/docker")
-    wrapper.write_text('''#!/usr/bin/env python3
-import os, sys
-args = sys.argv[1:]
-if args and args[0] in ('run', 'build') and '--network' not in args:
-    args[1:1] = ['--network', 'host']
-os.execv('/usr/bin/docker', ['docker', *args])
-''')
-    wrapper.chmod(0o755)
-    checked("bash", "-c", """set -euo pipefail
-export DEBIAN_FRONTEND=noninteractive
-apt-get update -qq
-apt-get install -y -qq curl gpg ca-certificates docker.io
-curl -fsSL https://pkg.cloudflareclient.com/pubkey.gpg | gpg --dearmor --yes -o /usr/share/keyrings/cloudflare-warp.gpg
-chmod 644 /usr/share/keyrings/cloudflare-warp.gpg
-. /etc/os-release
-echo "deb [signed-by=/usr/share/keyrings/cloudflare-warp.gpg] https://pkg.cloudflareclient.com/ $VERSION_CODENAME main" > /etc/apt/sources.list.d/cloudflare-client.list
-chmod 644 /etc/apt/sources.list.d/cloudflare-client.list
-apt-get update -qq
-apt-get install -y -qq cloudflare-warp
-update-alternatives --set iptables /usr/sbin/iptables-legacy
-update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
-""")
-    document = Element("dict")
-    for key, value in [("organization", "supabase"), ("auth_client_id", config["CLOUDFLARE_WARP_CLIENT_ID"]), ("auth_client_secret", config["CLOUDFLARE_WARP_CLIENT_SECRET"]), ("service_mode", "proxy"), ("warp_tunnel_protocol", "masque")]:
-        SubElement(document, "key").text = key
-        SubElement(document, "string").text = value
-    SubElement(document, "key").text = "proxy_port"
-    SubElement(document, "integer").text = "40000"
-    mdm = Path("/var/lib/cloudflare-warp/mdm.xml")
-    mdm.parent.mkdir(parents=True, exist_ok=True)
-    mdm.write_bytes(tostring(document))
-    mdm.chmod(0o600)
-
-
-def connect():
-    for _ in range(60):
-        registration = subprocess.run(["warp-cli", "--accept-tos", "registration", "show"], capture_output=True)
-        if registration.returncode == 0:
-            break
-        time.sleep(1)
-    checked("warp-cli", "--accept-tos", "connect")
-    for _ in range(60):
-        warp = subprocess.run(["warp-cli", "--accept-tos", "status"], capture_output=True, text=True)
-        docker = subprocess.run(["docker", "info"], capture_output=True)
-        if "Status update: Connected" in warp.stdout and docker.returncode == 0:
-            Path("/var/run/docker.sock").chmod(0o666)
-            return
-        time.sleep(1)
-    raise RuntimeError("WARP did not connect")
-
-
-def cleanup():
-    Path("/var/lib/cloudflare-warp/mdm.xml").unlink(missing_ok=True)
-    checked("warp-cli", "--accept-tos", "disconnect")
-    checked("warp-cli", "--accept-tos", "registration", "delete")
-    registration = subprocess.run(["warp-cli", "--accept-tos", "registration", "show"], capture_output=True)
-    if registration.returncode == 0:
-        raise RuntimeError("WARP registration still present")
-    print("WARP registration deleted")
-
-
-if __name__ == "__main__":
-    try:
-        if sys.argv[1:] == ["--self-check"]:
-            secret = "https://private.example/a?token=secret"
-            assert redact({"tool": "warp-client-secret", "project": "spark-project-key"}, ["warp-client-secret", "spark-project-key"]) == {"tool": "[REDACTED]", "project": "[REDACTED]"}
-            assert redact({"nested": [secret, quote(secret, safe=""), 4]}, [secret]) == {"nested": ["[REDACTED]", "[REDACTED]", 4]}
-        elif sys.argv[1:2] == ["sanitize"] and len(sys.argv) == 3:
-            sanitize(sys.argv[2])
-        elif sys.argv[1:] == ["cleanup"]:
-            cleanup()
-        elif sys.argv[1:] == ["connect"]:
-            connect()
-        elif not sys.argv[1:]:
-            setup()
-        else:
-            raise RuntimeError("Invalid arguments")
-    except Exception as error:
-        print(f"Spark setup error type {type(error).__name__} at line {traceback.extract_tb(error.__traceback__)[-1].lineno}", file=sys.stderr)
-        print("Spark setup or result sanitization failed. Inspect private Sandbox logs.", file=sys.stderr)
-        sys.exit(1)
diff --git a/apps/framework/scripts/setup-spark-warp.test.ts b/apps/framework/scripts/setup-spark-warp.test.ts
new file mode 100644
index 000000000..bcd00b27e
--- /dev/null
+++ b/apps/framework/scripts/setup-spark-warp.test.ts
@@ -0,0 +1,56 @@
+import { describe, expect, it } from 'vitest';
+import { sanitizeResult } from './setup-spark-warp.js';
+
+const accessToken = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ4In0.c2lnbmF0dXJl';
+const refreshToken = 'opaque-refresh+/token=';
+
+describe('Spark result sanitization', () => {
+  it('redacts configured values, encoded copies and object keys', () => {
+    const secret = 'https://private.example/a?token=secret';
+    const value = { [secret]: [secret, encodeURIComponent(secret), 4] };
+    expect(JSON.parse(sanitizeResult(value, [secret]))).toEqual({
+      '[REDACTED]': ['[REDACTED]', '[REDACTED]', 4],
+    });
+  });
+
+  it.each(['structured', 'stdout', 'escaped'])(
+    'discovers session tokens in %s responses and redacts every copy',
+    (format) => {
+      const session = {
+        access_token: accessToken,
+        refresh_token: refreshToken,
+        token_type: 'bearer',
+      };
+      const response =
+        format === 'structured'
+          ? session
+          : format === 'stdout'
+            ? `signup response: ${JSON.stringify(session)}`
+            : JSON.stringify(JSON.stringify(session));
+      const checks = [{ passed: false, notes: 'found: JWT-shaped token' }];
+      const value = {
+        copies: [accessToken, refreshToken, encodeURIComponent(refreshToken)],
+        response,
+        checks,
+      };
+      const payload = sanitizeResult(value, []);
+      for (const token of [accessToken, refreshToken]) {
+        expect(payload).not.toContain(token);
+        expect(payload).not.toContain(encodeURIComponent(token));
+      }
+      expect(JSON.parse(payload)).toMatchObject({
+        copies: ['[REDACTED]', '[REDACTED]', '[REDACTED]'],
+        checks,
+      });
+      expect(payload).toContain('bearer');
+    }
+  );
+
+  it('redacts JWTs printed without a session field', () => {
+    expect(
+      JSON.parse(sanitizeResult({ report: `Bearer ${accessToken}` }, []))
+    ).toEqual({
+      report: 'Bearer [REDACTED]',
+    });
+  });
+});
diff --git a/apps/framework/scripts/setup-spark-warp.ts b/apps/framework/scripts/setup-spark-warp.ts
new file mode 100644
index 000000000..31e7b7634
--- /dev/null
+++ b/apps/framework/scripts/setup-spark-warp.ts
@@ -0,0 +1,264 @@
+import { spawnSync } from 'node:child_process';
+import {
+  chmodSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { resolve } from 'node:path';
+import { setTimeout } from 'node:timers/promises';
+import { fileURLToPath } from 'node:url';
+
+const JWT = /eyJ[\w-]+\.[\w-]+\.[\w-]+/g;
+const SESSION_TOKEN =
+  /\\*["'](?:access_token|refresh_token|id_token)\\*["']\s*:\s*\\*["']([\w.~+/%=-]+)\\*["']/g;
+
+function sessionTokens(value: unknown): string[] {
+  if (typeof value === 'string') {
+    return [...value.matchAll(JWT)]
+      .map((match) => match[0])
+      .concat([...value.matchAll(SESSION_TOKEN)].map((match) => match[1]));
+  }
+  if (Array.isArray(value)) return value.flatMap(sessionTokens);
+  if (value && typeof value === 'object') {
+    return Object.entries(value).flatMap(([key, item]) => [
+      ...(['access_token', 'refresh_token', 'id_token'].includes(key) &&
+      typeof item === 'string'
+        ? [item]
+        : []),
+      ...sessionTokens(key),
+      ...sessionTokens(item),
+    ]);
+  }
+  return [];
+}
+
+function encodedSecret(secret: string): string {
+  return encodeURIComponent(secret).replace(
+    /[!'()*]/g,
+    (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`
+  );
+}
+
+function redact(value: unknown, secrets: readonly string[]): unknown {
+  if (typeof value === 'string') {
+    let text = value;
+    for (const secret of secrets) {
+      text = text
+        .replaceAll(secret, '[REDACTED]')
+        .replaceAll(encodedSecret(secret), '[REDACTED]');
+    }
+    return text;
+  }
+  if (Array.isArray(value)) return value.map((item) => redact(item, secrets));
+  if (value && typeof value === 'object') {
+    return Object.fromEntries(
+      Object.entries(value).map(([key, item]) => [
+        String(redact(key, secrets)),
+        redact(item, secrets),
+      ])
+    );
+  }
+  return value;
+}
+
+export function sanitizeResult(
+  value: unknown,
+  configuredSecrets: readonly string[]
+): string {
+  const secrets = [...new Set([...configuredSecrets, ...sessionTokens(value)])]
+    .filter(Boolean)
+    .sort((a, b) => b.length - a.length);
+  const payload = JSON.stringify(redact(value, secrets));
+  if (
+    secrets.some(
+      (secret) =>
+        payload.includes(secret) || payload.includes(encodedSecret(secret))
+    )
+  ) {
+    throw new Error('Result still contains a private value');
+  }
+  return payload;
+}
+
+function readJson(path: string): unknown {
+  return JSON.parse(readFileSync(path, 'utf8'));
+}
+
+function config() {
+  const value = readJson('/tmp/spark-warp-config.json');
+  if (!value || typeof value !== 'object' || Array.isArray(value))
+    throw new Error('Invalid Spark config');
+  function field(name: string): string {
+    const item = Object.entries(value ?? {}).find(([key]) => key === name)?.[1];
+    if (typeof item !== 'string' || !item)
+      throw new Error('Missing Spark config');
+    return item;
+  }
+  return {
+    apiUrl: field('SPARK_API_URL'),
+    clientId: field('CLOUDFLARE_WARP_CLIENT_ID'),
+    clientSecret: field('CLOUDFLARE_WARP_CLIENT_SECRET'),
+  };
+}
+
+function checked(command: string, args: string[] = []): string {
+  const result = spawnSync(command, args, {
+    encoding: 'utf8',
+    timeout: 300_000,
+    maxBuffer: 16 * 1024 * 1024,
+  });
+  if (result.status !== 0) {
+    writeFileSync(
+      '/tmp/spark-setup-error.log',
+      (result.stdout ?? '') + (result.stderr ?? ''),
+      { mode: 0o600 }
+    );
+    console.error(
+      `Spark setup executable ${command} failed with exit ${result.status}`
+    );
+    throw new Error('Spark setup command failed');
+  }
+  return result.stdout;
+}
+
+function setup() {
+  process.umask(0o077);
+  const settings = config();
+  // shortcut: Sandbox lacks bridge raw-table support, use host networking for Spark only.
+  writeFileSync(
+    '/usr/local/bin/docker',
+    `#!/bin/bash
+set -euo pipefail
+case "\${1:-}" in
+  run|build)
+    for arg in "$@"; do
+      case "$arg" in --network|--network=*) exec /usr/bin/docker "$@";; esac
+    done
+    command="$1"
+    shift
+    exec /usr/bin/docker "$command" --network host "$@"
+    ;;
+esac
+exec /usr/bin/docker "$@"
+`,
+    { mode: 0o755 }
+  );
+  checked('bash', [
+    '-c',
+    `set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+apt-get update -qq
+apt-get install -y -qq curl gpg ca-certificates docker.io
+curl -fsSL https://pkg.cloudflareclient.com/pubkey.gpg | gpg --dearmor --yes -o /usr/share/keyrings/cloudflare-warp.gpg
+chmod 644 /usr/share/keyrings/cloudflare-warp.gpg
+. /etc/os-release
+echo "deb [signed-by=/usr/share/keyrings/cloudflare-warp.gpg] https://pkg.cloudflareclient.com/ $VERSION_CODENAME main" > /etc/apt/sources.list.d/cloudflare-client.list
+chmod 644 /etc/apt/sources.list.d/cloudflare-client.list
+apt-get update -qq
+apt-get install -y -qq cloudflare-warp
+update-alternatives --set iptables /usr/sbin/iptables-legacy
+update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
+`,
+  ]);
+  const entries = Object.entries({
+    organization: 'supabase',
+    auth_client_id: settings.clientId,
+    auth_client_secret: settings.clientSecret,
+    service_mode: 'proxy',
+    warp_tunnel_protocol: 'masque',
+  }).map(
+    ([key, value]) =>
+      `<key>${key}</key><string>${value.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;')}</string>`
+  );
+  mkdirSync('/var/lib/cloudflare-warp', { recursive: true });
+  writeFileSync(
+    '/var/lib/cloudflare-warp/mdm.xml',
+    `<dict>${entries.join('')}<key>proxy_port</key><integer>40000</integer></dict>`,
+    { mode: 0o600 }
+  );
+}
+
+async function connect() {
+  for (let attempt = 0; attempt < 60; attempt++) {
+    if (
+      spawnSync('warp-cli', ['--accept-tos', 'registration', 'show']).status ===
+      0
+    )
+      break;
+    await setTimeout(1000);
+  }
+  checked('warp-cli', ['--accept-tos', 'connect']);
+  for (let attempt = 0; attempt < 60; attempt++) {
+    const warp = spawnSync('warp-cli', ['--accept-tos', 'status'], {
+      encoding: 'utf8',
+    });
+    const docker = spawnSync('docker', ['info']);
+    if (
+      warp.stdout?.includes('Status update: Connected') &&
+      docker.status === 0
+    ) {
+      chmodSync('/var/run/docker.sock', 0o666);
+      return;
+    }
+    await setTimeout(1000);
+  }
+  throw new Error('WARP did not connect');
+}
+
+function sanitize(path: string) {
+  const settings = config();
+  const secrets = readJson('/tmp/spark-redactions.json');
+  if (
+    !Array.isArray(secrets) ||
+    !secrets.every((item): item is string => typeof item === 'string')
+  )
+    throw new Error('Invalid Spark redactions');
+  writeFileSync(
+    path,
+    sanitizeResult(readJson(path), [
+      ...Object.values(settings),
+      new URL(settings.apiUrl).hostname,
+      ...secrets,
+    ])
+  );
+}
+
+function cleanup() {
+  rmSync('/var/lib/cloudflare-warp/mdm.xml', { force: true });
+  checked('warp-cli', ['--accept-tos', 'disconnect']);
+  checked('warp-cli', ['--accept-tos', 'registration', 'delete']);
+  if (
+    spawnSync('warp-cli', ['--accept-tos', 'registration', 'show']).status === 0
+  )
+    throw new Error('WARP registration still present');
+  console.log('WARP registration deleted');
+}
+
+async function main(args: string[]) {
+  if (args.length === 0) setup();
+  else if (args.length === 1 && args[0] === 'connect') await connect();
+  else if (args.length === 1 && args[0] === 'cleanup') cleanup();
+  else if (args.length === 2 && args[0] === 'sanitize') sanitize(args[1]);
+  else throw new Error('Invalid arguments');
+}
+
+if (
+  process.argv[1] &&
+  resolve(process.argv[1]) === fileURLToPath(import.meta.url)
+) {
+  main(process.argv.slice(2)).catch((error: unknown) => {
+    const type =
+      error instanceof Error ? error.name.replace(/[^A-Za-z]/g, '') : 'Error';
+    const line =
+      error instanceof Error
+        ? (error.stack?.match(/setup-spark-warp\.ts:(\d+)/)?.[1] ?? '0')
+        : '0';
+    console.error(`Spark setup error type ${type} at line ${line}`);
+    console.error(
+      'Spark setup or result sanitization failed. Inspect private Sandbox logs.'
+    );
+    process.exitCode = 1;
+  });
+}

From a446bcfbad5354416a2bf6ea77096726bc2ee343 Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Thu, 12 Nov 2026 15:17:48 -0500
Subject: [PATCH 4/6] refactor: simplify Spark CI setup

---
 CONTRIBUTING.md                               |   4 +
 README.md                                     |  25 --
 ...rp.test.ts => redact-spark-result.test.ts} |   2 +-
 apps/framework/scripts/redact-spark-result.ts | 105 +++++++
 .../scripts/run-vercel-evals.test.ts          |  96 +++----
 apps/framework/scripts/run-vercel-evals.ts    | 168 +++++++----
 apps/framework/scripts/setup-spark-warp.ts    | 264 ------------------
 7 files changed, 274 insertions(+), 390 deletions(-)
 rename apps/framework/scripts/{setup-spark-warp.test.ts => redact-spark-result.test.ts} (96%)
 create mode 100644 apps/framework/scripts/redact-spark-result.ts
 delete mode 100644 apps/framework/scripts/setup-spark-warp.ts

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 1b725930c..98dac992f 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -46,6 +46,10 @@ Add a `*.experiment.ts` file under `experiments/<suite>/` for the agent, model,
 
 The suite folder decides which evals the experiment runs: `experiments/<suite>/` runs the evals in `evals/<suite>/`. If this experiment should be part of our published benchmark, put it in `experiments/benchmark/` and include a corresponding `*-no-skills` variant (`skills: []`) to compare results with and without skills. To run one experiment in several suites, export a shared preset from `experiments/presets.ts` and spread it in a file under each suite folder.
 
+## Spark evals
+
+Local runs need `SPARK_API_URL` and WARP access. For CI, configure `SPARK_API_URL`, `CLOUDFLARE_WARP_CLIENT_ID`, and `CLOUDFLARE_WARP_CLIENT_SECRET` as repository secrets alongside the existing Vercel and provider credentials.
+
 ## Submitting evals for review
 
 Before submitting an eval for review, try running it locally to sanity check that it can complete without errors. It's okay if agents fail the eval, we just don't want them to be scored unfairly for framework limitations.
diff --git a/README.md b/README.md
index 1c22f2d4e..b859d7ee7 100644
--- a/README.md
+++ b/README.md
@@ -74,31 +74,6 @@ Run every experiment in `experiments/benchmark/` across all benchmark evals:
 pnpm eval -- --suite benchmark
 ```
 
-### Spark in CI
-
-Dispatch the normal workflow with the Spark suite:
-
-```bash
-gh workflow run eval-refresh.yml -R supabase/evals \
-  -f suite=spark -f experiments=codex-gpt-6-luna \
-  -f eval=build-database-001-task-due-dates -f runs=1 \
-  -f commit_results=none
-```
-
-The runner needs `SPARK_API_URL`, `CLOUDFLARE_WARP_CLIENT_ID`, and
-`CLOUDFLARE_WARP_CLIENT_SECRET` in repository secrets, alongside the existing
-Vercel and provider credentials. Only Spark uses the Ubuntu universal image,
-WARP's local HTTP proxy on port 40000, and Docker host networking. Provider
-requests bypass WARP and retain Sandbox credential brokering.
-Spark agents do not receive the host Docker socket. The runner does not
-restrict WARP's proxy to Spark hosts.
-
-Spark results are sanitized inside Sandbox before download. Public artifacts
-contain result JSON without agent workspaces or binary session archives.
-Raw eval output stays inside Sandbox. [Cloudflare documents a 10-second proxy
-request limit](https://developers.cloudflare.com/cloudflare-one/team-and-resources/devices/cloudflare-one-client/configure/modes/#local-proxy-mode), so longer Spark requests may fail. Projects currently remain
-allocated after a run because a deletion endpoint is not implemented.
-
 ### View results in the web app
 
 Set `BRAINTRUST_API_KEY` and `BRAINTRUST_PROJECT_ID` in the repository's `.env`, then run:
diff --git a/apps/framework/scripts/setup-spark-warp.test.ts b/apps/framework/scripts/redact-spark-result.test.ts
similarity index 96%
rename from apps/framework/scripts/setup-spark-warp.test.ts
rename to apps/framework/scripts/redact-spark-result.test.ts
index bcd00b27e..3e9aebf5e 100644
--- a/apps/framework/scripts/setup-spark-warp.test.ts
+++ b/apps/framework/scripts/redact-spark-result.test.ts
@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest';
-import { sanitizeResult } from './setup-spark-warp.js';
+import { sanitizeResult } from './redact-spark-result.js';
 
 const accessToken = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ4In0.c2lnbmF0dXJl';
 const refreshToken = 'opaque-refresh+/token=';
diff --git a/apps/framework/scripts/redact-spark-result.ts b/apps/framework/scripts/redact-spark-result.ts
new file mode 100644
index 000000000..d5f336765
--- /dev/null
+++ b/apps/framework/scripts/redact-spark-result.ts
@@ -0,0 +1,105 @@
+import { readFileSync, writeFileSync } from 'node:fs';
+
+const JWT = /eyJ[\w-]+\.[\w-]+\.[\w-]+/g;
+const SESSION_TOKEN =
+  /\\*["'](?:access_token|refresh_token|id_token)\\*["']\s*:\s*\\*["']([\w.~+/%=-]+)\\*["']/g;
+
+function sessionTokens(value: unknown): string[] {
+  if (typeof value === 'string') {
+    return [...value.matchAll(JWT)]
+      .map((match) => match[0])
+      .concat([...value.matchAll(SESSION_TOKEN)].map((match) => match[1]));
+  }
+  if (Array.isArray(value)) return value.flatMap(sessionTokens);
+  if (value && typeof value === 'object') {
+    return Object.entries(value).flatMap(([key, item]) => [
+      ...(['access_token', 'refresh_token', 'id_token'].includes(key) &&
+      typeof item === 'string'
+        ? [item]
+        : []),
+      ...sessionTokens(key),
+      ...sessionTokens(item),
+    ]);
+  }
+  return [];
+}
+
+function encodedSecret(secret: string): string {
+  return encodeURIComponent(secret).replace(
+    /[!'()*]/g,
+    (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`
+  );
+}
+
+function redact(value: unknown, secrets: readonly string[]): unknown {
+  if (typeof value === 'string') {
+    let text = value;
+    for (const secret of secrets) {
+      text = text
+        .replaceAll(secret, '[REDACTED]')
+        .replaceAll(encodedSecret(secret), '[REDACTED]');
+    }
+    return text;
+  }
+  if (Array.isArray(value)) return value.map((item) => redact(item, secrets));
+  if (value && typeof value === 'object') {
+    return Object.fromEntries(
+      Object.entries(value).map(([key, item]) => [
+        String(redact(key, secrets)),
+        redact(item, secrets),
+      ])
+    );
+  }
+  return value;
+}
+
+export function sanitizeResult(
+  value: unknown,
+  configuredSecrets: readonly string[]
+): string {
+  const secrets = [...new Set([...configuredSecrets, ...sessionTokens(value)])]
+    .filter(Boolean)
+    .sort((a, b) => b.length - a.length);
+  const payload = JSON.stringify(redact(value, secrets));
+  if (
+    secrets.some(
+      (secret) =>
+        payload.includes(secret) || payload.includes(encodedSecret(secret))
+    )
+  ) {
+    throw new Error('Result still contains a private value');
+  }
+  return payload;
+}
+
+if (import.meta.main) {
+  try {
+    const secrets = [
+      '/tmp/spark-warp-secrets.json',
+      '/tmp/spark-redactions.json',
+    ].flatMap((path) => {
+      const value: unknown = JSON.parse(readFileSync(path, 'utf8'));
+      if (
+        !Array.isArray(value) ||
+        !value.every((item): item is string => typeof item === 'string')
+      )
+        throw new Error('Invalid Spark redactions');
+      return value;
+    });
+    const path = process.argv[2];
+    const value: unknown = JSON.parse(readFileSync(path, 'utf8'));
+    writeFileSync(path, sanitizeResult(value, secrets));
+  } catch (error) {
+    writeFileSync(
+      '/tmp/spark-redaction-error.log',
+      error instanceof Error
+        ? (error.stack ?? error.message)
+        : 'Redaction failed',
+      { mode: 0o600 }
+    );
+    console.error(
+      'Spark result redaction failed. Inspect private Sandbox logs.'
+    );
+    process.exitCode = 1;
+  }
+}
diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts
index f35e3d9f2..7ed7ba1cc 100644
--- a/apps/framework/scripts/run-vercel-evals.test.ts
+++ b/apps/framework/scripts/run-vercel-evals.test.ts
@@ -34,12 +34,22 @@ import {
   tagValue,
   expandJobs,
   type EvalPair,
+  sparkWarpMdm,
 } from './run-vercel-evals.js';
 
 vi.mock('@supabase-evals/sandbox', () => ({ resolveCliVersion: vi.fn() }));
 
 const BROKERED_NAMES = BROKERED_KEYS.map(({ name }) => name);
 
+it('escapes credentials as XML text in the WARP configuration', () => {
+  expect(sparkWarpMdm('client<&>', 'secret<&>')).toContain(
+    '<key>auth_client_id</key><string>client&lt;&amp;&gt;</string>'
+  );
+  expect(sparkWarpMdm('client<&>', 'secret<&>')).toContain(
+    '<key>auth_client_secret</key><string>secret&lt;&amp;&gt;</string>'
+  );
+});
+
 describe('Spark terminal failures', () => {
   const options = {
     pair: {
@@ -66,33 +76,28 @@ describe('Spark terminal failures', () => {
     vi.stubEnv('SPARK_API_URL', 'https://control.example');
     vi.stubEnv('CLOUDFLARE_WARP_CLIENT_ID', 'test.access');
     vi.stubEnv('CLOUDFLARE_WARP_CLIENT_SECRET', 'test');
-    vi.spyOn(console, 'warn').mockImplementation(() => {});
   });
   afterEach(() => {
     vi.unstubAllEnvs();
     vi.restoreAllMocks();
   });
 
-  it('preserves terminal create failures so the outer retry loop stops', async () => {
-    const create = vi
-      .spyOn(Sandbox, 'create')
-      .mockRejectedValue(new APIError(new Response(null, { status: 401 })));
-    await expect(runPairOnce(options, credentials)).rejects.toBeInstanceOf(
-      AbortError
-    );
-    expect(create).toHaveBeenCalledTimes(1);
-  });
-
-  it('does not retry missing credentials or allocate a VM', async () => {
-    vi.stubEnv('CLOUDFLARE_WARP_CLIENT_SECRET', '');
-    const create = vi
-      .spyOn(Sandbox, 'create')
-      .mockRejectedValue(new APIError(new Response(null, { status: 401 })));
-    await expect(runPairOnce(options, credentials)).rejects.toBeInstanceOf(
-      AbortError
-    );
-    expect(create).not.toHaveBeenCalled();
-  });
+  it.each(['unauthorized Sandbox', 'missing credentials'])(
+    'aborts %s without retrying',
+    async (reason) => {
+      if (reason === 'missing credentials')
+        vi.stubEnv('CLOUDFLARE_WARP_CLIENT_SECRET', '');
+      const create = vi
+        .spyOn(Sandbox, 'create')
+        .mockRejectedValue(new APIError(new Response(null, { status: 401 })));
+      await expect(runPairOnce(options, credentials)).rejects.toBeInstanceOf(
+        AbortError
+      );
+      expect(create).toHaveBeenCalledTimes(
+        reason === 'missing credentials' ? 0 : 1
+      );
+    }
+  );
 });
 
 describe('agentEnvironment', () => {
@@ -480,37 +485,24 @@ describe('Vercel eval controller', () => {
   });
 
   it('downloads only sanitized JSON for Spark, never workspace or session archives', async () => {
-    const temporary = mkdtempSync(join(tmpdir(), 'spark-download-test-'));
-    const paths: string[] = [];
-    try {
-      await downloadResults(
-        {
-          downloadFile: async (source, destination) => {
-            paths.push(source.path);
-            mkdirSync(dirname(destination.path), { recursive: true });
-            writeFileSync(destination.path, '{"passed":true}');
-            return destination.path;
-          },
-        },
-        { eval_id: 'eval-1', experiment: 'experiment-1', eval_suite: 'spark' },
-        1,
-        temporary,
-        '/vercel/evals'
-      );
-      expect(paths).toEqual([
-        '/vercel/evals/results/spark/experiment-1/eval-1/run-1/result.json',
-      ]);
-      expect(
-        readdirSync(
-          join(
-            temporary,
-            'raw-results-spark__experiment-1__eval-1/eval-1/run-1'
-          )
-        )
-      ).toEqual(['result.json.partial']);
-    } finally {
-      rmSync(temporary, { recursive: true, force: true });
-    }
+    const downloadFile = vi.fn().mockResolvedValue('downloaded');
+    const result = await downloadResults(
+      { downloadFile },
+      { eval_id: 'eval-1', experiment: 'experiment-1', eval_suite: 'spark' },
+      1,
+      '/tmp/unused-spark-results',
+      '/vercel/evals'
+    );
+    expect(downloadFile).toHaveBeenCalledExactlyOnceWith(
+      {
+        path: '/vercel/evals/results/spark/experiment-1/eval-1/run-1/result.json',
+      },
+      { path: result.partialPath },
+      { mkdirRecursive: true }
+    );
+    expect(result.partialPath).toBe(
+      '/tmp/unused-spark-results/raw-results-spark__experiment-1__eval-1/eval-1/run-1/result.json.partial'
+    );
   });
 
   it('downloads result metadata separately from agent workspace files', async () => {
diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts
index 4434719d1..1d2003ebd 100644
--- a/apps/framework/scripts/run-vercel-evals.ts
+++ b/apps/framework/scripts/run-vercel-evals.ts
@@ -338,6 +338,20 @@ async function runPairs(options: RunnerOptions): Promise<void> {
   }
 }
 
+export function sparkWarpMdm(clientId: string, clientSecret: string): string {
+  const entries = Object.entries({
+    organization: 'supabase',
+    auth_client_id: clientId,
+    auth_client_secret: clientSecret,
+    service_mode: 'proxy',
+    warp_tunnel_protocol: 'masque',
+  }).map(
+    ([key, value]) =>
+      `<key>${key}</key><string>${value.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;')}</string>`
+  );
+  return `<dict>${entries.join('')}<key>proxy_port</key><integer>40000</integer></dict>`;
+}
+
 export async function runPairOnce(
   options: PairOptions,
   credentials: VercelCredentials
@@ -411,45 +425,102 @@ export async function runPairOnce(
     if (spark) {
       await sandbox.writeFiles([
         {
-          path: '/tmp/setup-spark-warp.ts',
+          path: '/tmp/redact-spark-result.ts',
           content: readFileSync(
-            new URL('./setup-spark-warp.ts', import.meta.url)
+            new URL('./redact-spark-result.ts', import.meta.url)
           ),
         },
         {
-          path: '/tmp/spark-warp-config.json',
-          content: Buffer.from(JSON.stringify(sparkConfig)),
+          path: '/tmp/spark-warp-secrets.json',
+          content: Buffer.from(
+            JSON.stringify([
+              ...Object.values(sparkConfig),
+              new URL(sparkConfig.SPARK_API_URL).hostname,
+            ])
+          ),
           mode: 0o600,
         },
-      ]);
-      await runSandboxCommand(
-        sandbox,
-        label,
-        'install Spark WARP proxy',
         {
-          cmd: 'node',
-          args: ['/tmp/setup-spark-warp.ts'],
-          sudo: true,
-          timeoutMs: 10 * 60 * 1_000,
+          path: '/tmp/spark-warp-mdm.xml',
+          content: Buffer.from(
+            sparkWarpMdm(
+              sparkConfig.CLOUDFLARE_WARP_CLIENT_ID,
+              sparkConfig.CLOUDFLARE_WARP_CLIENT_SECRET
+            )
+          ),
+          mode: 0o600,
         },
-        true
-      );
+      ]);
+      await runSandboxCommand(sandbox, label, 'install Spark WARP proxy', {
+        cmd: 'bash',
+        args: [
+          '-euo',
+          'pipefail',
+          '-c',
+          `umask 077
+exec >>/tmp/spark-setup.log 2>&1
+export DEBIAN_FRONTEND=noninteractive
+apt-get update -qq
+apt-get install -y -qq curl gpg ca-certificates docker.io
+curl -fsSL https://pkg.cloudflareclient.com/pubkey.gpg | gpg --dearmor --yes -o /usr/share/keyrings/cloudflare-warp.gpg
+chmod 644 /usr/share/keyrings/cloudflare-warp.gpg
+. /etc/os-release
+echo "deb [signed-by=/usr/share/keyrings/cloudflare-warp.gpg] https://pkg.cloudflareclient.com/ $VERSION_CODENAME main" > /etc/apt/sources.list.d/cloudflare-client.list
+chmod 644 /etc/apt/sources.list.d/cloudflare-client.list
+apt-get update -qq
+apt-get install -y -qq cloudflare-warp
+update-alternatives --set iptables /usr/sbin/iptables-legacy
+update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
+install -D -m 600 /tmp/spark-warp-mdm.xml /var/lib/cloudflare-warp/mdm.xml
+# shortcut: Sandbox lacks bridge raw-table support, use host networking for Spark only.
+cat > /usr/local/bin/docker <<'DOCKER'
+#!/bin/bash
+set -euo pipefail
+case "\${1:-}" in
+  run|build)
+    for arg in "$@"; do
+      case "$arg" in --network|--network=*) exec /usr/bin/docker "$@";; esac
+    done
+    exec /usr/bin/docker "$1" --network host "\${@:2}"
+    ;;
+esac
+exec /usr/bin/docker "$@"
+DOCKER
+chmod 755 /usr/local/bin/docker`,
+        ],
+        sudo: true,
+        timeoutMs: 10 * 60 * 1_000,
+      });
       sparkSetupInstalled = true;
       for (const cmd of ['warp-svc', 'dockerd']) {
         await sandbox.runCommand({ cmd, sudo: true, detached: true });
       }
-      await runSandboxCommand(
-        sandbox,
-        label,
-        'connect Spark WARP proxy',
-        {
-          cmd: 'node',
-          args: ['/tmp/setup-spark-warp.ts', 'connect'],
-          sudo: true,
-          timeoutMs: 3 * 60 * 1_000,
-        },
-        true
-      );
+      await runSandboxCommand(sandbox, label, 'connect Spark WARP proxy', {
+        cmd: 'bash',
+        args: [
+          '-euo',
+          'pipefail',
+          '-c',
+          `umask 077
+exec >>/tmp/spark-setup.log 2>&1
+for attempt in {1..60}; do
+  warp-cli --accept-tos registration show >/dev/null 2>&1 && break
+  sleep 1
+done
+warp-cli --accept-tos connect
+for attempt in {1..60}; do
+  if warp-cli --accept-tos status | grep -F 'Status update: Connected' >/dev/null && docker info >/dev/null 2>&1; then
+    chmod 666 /var/run/docker.sock
+    exit 0
+  fi
+  sleep 1
+done
+echo 'WARP did not connect' >&2
+exit 1`,
+        ],
+        sudo: true,
+        timeoutMs: 3 * 60 * 1_000,
+      });
     } else {
       await runSandboxCommand(sandbox, label, 'install Docker', {
         cmd: 'dnf',
@@ -552,8 +623,7 @@ export async function runPairOnce(
       await runSandboxCommand(sandbox, label, 'sanitize Spark result', {
         cmd: 'node',
         args: [
-          '/tmp/setup-spark-warp.ts',
-          'sanitize',
+          '/tmp/redact-spark-result.ts',
           `${repoCwd}/results/${pair.eval_suite}/${pair.experiment}/${pair.eval_id}/run-${run}/result.json`,
         ],
         sudo: true,
@@ -584,14 +654,8 @@ export async function runPairOnce(
     if (spark) {
       const step =
         error instanceof SandboxCommandError ? error.step : 'setup or download';
-      const receipt =
-        error instanceof Error
-          ? error.message
-              .match(/Spark setup error type [A-Za-z]+ at line [0-9]+/g)
-              ?.join(', ')
-          : undefined;
       const safeError = new Error(
-        `Spark ${step} failed. ${receipt ?? 'Inspect private Sandbox logs.'}`
+        `Spark ${step} failed. Inspect private Sandbox logs.`
       );
       throw error instanceof AbortError ? new AbortError(safeError) : safeError;
     }
@@ -601,18 +665,26 @@ export async function runPairOnce(
     if (sandbox) {
       if (sparkSetupInstalled) {
         try {
-          await runSandboxCommand(
-            sandbox,
-            label,
-            'deregister WARP',
-            {
-              cmd: 'node',
-              args: ['/tmp/setup-spark-warp.ts', 'cleanup'],
-              sudo: true,
-              timeoutMs: 30_000,
-            },
-            true
-          );
+          await runSandboxCommand(sandbox, label, 'deregister WARP', {
+            cmd: 'bash',
+            args: [
+              '-euo',
+              'pipefail',
+              '-c',
+              `umask 077
+exec >>/tmp/spark-setup.log 2>&1
+rm -f /var/lib/cloudflare-warp/mdm.xml
+warp-cli --accept-tos disconnect
+warp-cli --accept-tos registration delete
+if warp-cli --accept-tos registration show >/dev/null 2>&1; then
+  echo 'WARP registration still present' >&2
+  exit 1
+fi`,
+            ],
+            sudo: true,
+            timeoutMs: 30_000,
+          });
+          console.log(`${label} WARP registration deleted`);
         } catch {
           console.warn(
             `${label} WARP deregistration failed. Continuing VM cleanup.`
diff --git a/apps/framework/scripts/setup-spark-warp.ts b/apps/framework/scripts/setup-spark-warp.ts
deleted file mode 100644
index 31e7b7634..000000000
--- a/apps/framework/scripts/setup-spark-warp.ts
+++ /dev/null
@@ -1,264 +0,0 @@
-import { spawnSync } from 'node:child_process';
-import {
-  chmodSync,
-  mkdirSync,
-  readFileSync,
-  rmSync,
-  writeFileSync,
-} from 'node:fs';
-import { resolve } from 'node:path';
-import { setTimeout } from 'node:timers/promises';
-import { fileURLToPath } from 'node:url';
-
-const JWT = /eyJ[\w-]+\.[\w-]+\.[\w-]+/g;
-const SESSION_TOKEN =
-  /\\*["'](?:access_token|refresh_token|id_token)\\*["']\s*:\s*\\*["']([\w.~+/%=-]+)\\*["']/g;
-
-function sessionTokens(value: unknown): string[] {
-  if (typeof value === 'string') {
-    return [...value.matchAll(JWT)]
-      .map((match) => match[0])
-      .concat([...value.matchAll(SESSION_TOKEN)].map((match) => match[1]));
-  }
-  if (Array.isArray(value)) return value.flatMap(sessionTokens);
-  if (value && typeof value === 'object') {
-    return Object.entries(value).flatMap(([key, item]) => [
-      ...(['access_token', 'refresh_token', 'id_token'].includes(key) &&
-      typeof item === 'string'
-        ? [item]
-        : []),
-      ...sessionTokens(key),
-      ...sessionTokens(item),
-    ]);
-  }
-  return [];
-}
-
-function encodedSecret(secret: string): string {
-  return encodeURIComponent(secret).replace(
-    /[!'()*]/g,
-    (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`
-  );
-}
-
-function redact(value: unknown, secrets: readonly string[]): unknown {
-  if (typeof value === 'string') {
-    let text = value;
-    for (const secret of secrets) {
-      text = text
-        .replaceAll(secret, '[REDACTED]')
-        .replaceAll(encodedSecret(secret), '[REDACTED]');
-    }
-    return text;
-  }
-  if (Array.isArray(value)) return value.map((item) => redact(item, secrets));
-  if (value && typeof value === 'object') {
-    return Object.fromEntries(
-      Object.entries(value).map(([key, item]) => [
-        String(redact(key, secrets)),
-        redact(item, secrets),
-      ])
-    );
-  }
-  return value;
-}
-
-export function sanitizeResult(
-  value: unknown,
-  configuredSecrets: readonly string[]
-): string {
-  const secrets = [...new Set([...configuredSecrets, ...sessionTokens(value)])]
-    .filter(Boolean)
-    .sort((a, b) => b.length - a.length);
-  const payload = JSON.stringify(redact(value, secrets));
-  if (
-    secrets.some(
-      (secret) =>
-        payload.includes(secret) || payload.includes(encodedSecret(secret))
-    )
-  ) {
-    throw new Error('Result still contains a private value');
-  }
-  return payload;
-}
-
-function readJson(path: string): unknown {
-  return JSON.parse(readFileSync(path, 'utf8'));
-}
-
-function config() {
-  const value = readJson('/tmp/spark-warp-config.json');
-  if (!value || typeof value !== 'object' || Array.isArray(value))
-    throw new Error('Invalid Spark config');
-  function field(name: string): string {
-    const item = Object.entries(value ?? {}).find(([key]) => key === name)?.[1];
-    if (typeof item !== 'string' || !item)
-      throw new Error('Missing Spark config');
-    return item;
-  }
-  return {
-    apiUrl: field('SPARK_API_URL'),
-    clientId: field('CLOUDFLARE_WARP_CLIENT_ID'),
-    clientSecret: field('CLOUDFLARE_WARP_CLIENT_SECRET'),
-  };
-}
-
-function checked(command: string, args: string[] = []): string {
-  const result = spawnSync(command, args, {
-    encoding: 'utf8',
-    timeout: 300_000,
-    maxBuffer: 16 * 1024 * 1024,
-  });
-  if (result.status !== 0) {
-    writeFileSync(
-      '/tmp/spark-setup-error.log',
-      (result.stdout ?? '') + (result.stderr ?? ''),
-      { mode: 0o600 }
-    );
-    console.error(
-      `Spark setup executable ${command} failed with exit ${result.status}`
-    );
-    throw new Error('Spark setup command failed');
-  }
-  return result.stdout;
-}
-
-function setup() {
-  process.umask(0o077);
-  const settings = config();
-  // shortcut: Sandbox lacks bridge raw-table support, use host networking for Spark only.
-  writeFileSync(
-    '/usr/local/bin/docker',
-    `#!/bin/bash
-set -euo pipefail
-case "\${1:-}" in
-  run|build)
-    for arg in "$@"; do
-      case "$arg" in --network|--network=*) exec /usr/bin/docker "$@";; esac
-    done
-    command="$1"
-    shift
-    exec /usr/bin/docker "$command" --network host "$@"
-    ;;
-esac
-exec /usr/bin/docker "$@"
-`,
-    { mode: 0o755 }
-  );
-  checked('bash', [
-    '-c',
-    `set -euo pipefail
-export DEBIAN_FRONTEND=noninteractive
-apt-get update -qq
-apt-get install -y -qq curl gpg ca-certificates docker.io
-curl -fsSL https://pkg.cloudflareclient.com/pubkey.gpg | gpg --dearmor --yes -o /usr/share/keyrings/cloudflare-warp.gpg
-chmod 644 /usr/share/keyrings/cloudflare-warp.gpg
-. /etc/os-release
-echo "deb [signed-by=/usr/share/keyrings/cloudflare-warp.gpg] https://pkg.cloudflareclient.com/ $VERSION_CODENAME main" > /etc/apt/sources.list.d/cloudflare-client.list
-chmod 644 /etc/apt/sources.list.d/cloudflare-client.list
-apt-get update -qq
-apt-get install -y -qq cloudflare-warp
-update-alternatives --set iptables /usr/sbin/iptables-legacy
-update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
-`,
-  ]);
-  const entries = Object.entries({
-    organization: 'supabase',
-    auth_client_id: settings.clientId,
-    auth_client_secret: settings.clientSecret,
-    service_mode: 'proxy',
-    warp_tunnel_protocol: 'masque',
-  }).map(
-    ([key, value]) =>
-      `<key>${key}</key><string>${value.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;')}</string>`
-  );
-  mkdirSync('/var/lib/cloudflare-warp', { recursive: true });
-  writeFileSync(
-    '/var/lib/cloudflare-warp/mdm.xml',
-    `<dict>${entries.join('')}<key>proxy_port</key><integer>40000</integer></dict>`,
-    { mode: 0o600 }
-  );
-}
-
-async function connect() {
-  for (let attempt = 0; attempt < 60; attempt++) {
-    if (
-      spawnSync('warp-cli', ['--accept-tos', 'registration', 'show']).status ===
-      0
-    )
-      break;
-    await setTimeout(1000);
-  }
-  checked('warp-cli', ['--accept-tos', 'connect']);
-  for (let attempt = 0; attempt < 60; attempt++) {
-    const warp = spawnSync('warp-cli', ['--accept-tos', 'status'], {
-      encoding: 'utf8',
-    });
-    const docker = spawnSync('docker', ['info']);
-    if (
-      warp.stdout?.includes('Status update: Connected') &&
-      docker.status === 0
-    ) {
-      chmodSync('/var/run/docker.sock', 0o666);
-      return;
-    }
-    await setTimeout(1000);
-  }
-  throw new Error('WARP did not connect');
-}
-
-function sanitize(path: string) {
-  const settings = config();
-  const secrets = readJson('/tmp/spark-redactions.json');
-  if (
-    !Array.isArray(secrets) ||
-    !secrets.every((item): item is string => typeof item === 'string')
-  )
-    throw new Error('Invalid Spark redactions');
-  writeFileSync(
-    path,
-    sanitizeResult(readJson(path), [
-      ...Object.values(settings),
-      new URL(settings.apiUrl).hostname,
-      ...secrets,
-    ])
-  );
-}
-
-function cleanup() {
-  rmSync('/var/lib/cloudflare-warp/mdm.xml', { force: true });
-  checked('warp-cli', ['--accept-tos', 'disconnect']);
-  checked('warp-cli', ['--accept-tos', 'registration', 'delete']);
-  if (
-    spawnSync('warp-cli', ['--accept-tos', 'registration', 'show']).status === 0
-  )
-    throw new Error('WARP registration still present');
-  console.log('WARP registration deleted');
-}
-
-async function main(args: string[]) {
-  if (args.length === 0) setup();
-  else if (args.length === 1 && args[0] === 'connect') await connect();
-  else if (args.length === 1 && args[0] === 'cleanup') cleanup();
-  else if (args.length === 2 && args[0] === 'sanitize') sanitize(args[1]);
-  else throw new Error('Invalid arguments');
-}
-
-if (
-  process.argv[1] &&
-  resolve(process.argv[1]) === fileURLToPath(import.meta.url)
-) {
-  main(process.argv.slice(2)).catch((error: unknown) => {
-    const type =
-      error instanceof Error ? error.name.replace(/[^A-Za-z]/g, '') : 'Error';
-    const line =
-      error instanceof Error
-        ? (error.stack?.match(/setup-spark-warp\.ts:(\d+)/)?.[1] ?? '0')
-        : '0';
-    console.error(`Spark setup error type ${type} at line ${line}`);
-    console.error(
-      'Spark setup or result sanitization failed. Inspect private Sandbox logs.'
-    );
-    process.exitCode = 1;
-  });
-}

From 1e75dbc1c79e4627da38fa3e8166e64e8197e642 Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Thu, 12 Nov 2026 15:33:50 -0500
Subject: [PATCH 5/6] refactor: extract Spark setup script

---
 apps/framework/scripts/run-vercel-evals.ts | 34 +++-------------------
 apps/framework/scripts/setup-spark-warp.sh | 31 ++++++++++++++++++++
 2 files changed, 35 insertions(+), 30 deletions(-)
 create mode 100644 apps/framework/scripts/setup-spark-warp.sh

diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts
index 1d2003ebd..925f2ad35 100644
--- a/apps/framework/scripts/run-vercel-evals.ts
+++ b/apps/framework/scripts/run-vercel-evals.ts
@@ -457,36 +457,10 @@ export async function runPairOnce(
           '-euo',
           'pipefail',
           '-c',
-          `umask 077
-exec >>/tmp/spark-setup.log 2>&1
-export DEBIAN_FRONTEND=noninteractive
-apt-get update -qq
-apt-get install -y -qq curl gpg ca-certificates docker.io
-curl -fsSL https://pkg.cloudflareclient.com/pubkey.gpg | gpg --dearmor --yes -o /usr/share/keyrings/cloudflare-warp.gpg
-chmod 644 /usr/share/keyrings/cloudflare-warp.gpg
-. /etc/os-release
-echo "deb [signed-by=/usr/share/keyrings/cloudflare-warp.gpg] https://pkg.cloudflareclient.com/ $VERSION_CODENAME main" > /etc/apt/sources.list.d/cloudflare-client.list
-chmod 644 /etc/apt/sources.list.d/cloudflare-client.list
-apt-get update -qq
-apt-get install -y -qq cloudflare-warp
-update-alternatives --set iptables /usr/sbin/iptables-legacy
-update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
-install -D -m 600 /tmp/spark-warp-mdm.xml /var/lib/cloudflare-warp/mdm.xml
-# shortcut: Sandbox lacks bridge raw-table support, use host networking for Spark only.
-cat > /usr/local/bin/docker <<'DOCKER'
-#!/bin/bash
-set -euo pipefail
-case "\${1:-}" in
-  run|build)
-    for arg in "$@"; do
-      case "$arg" in --network|--network=*) exec /usr/bin/docker "$@";; esac
-    done
-    exec /usr/bin/docker "$1" --network host "\${@:2}"
-    ;;
-esac
-exec /usr/bin/docker "$@"
-DOCKER
-chmod 755 /usr/local/bin/docker`,
+          readFileSync(
+            new URL('./setup-spark-warp.sh', import.meta.url),
+            'utf8'
+          ),
         ],
         sudo: true,
         timeoutMs: 10 * 60 * 1_000,
diff --git a/apps/framework/scripts/setup-spark-warp.sh b/apps/framework/scripts/setup-spark-warp.sh
new file mode 100644
index 000000000..44b0e62fb
--- /dev/null
+++ b/apps/framework/scripts/setup-spark-warp.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+umask 077
+exec >>/tmp/spark-setup.log 2>&1
+export DEBIAN_FRONTEND=noninteractive
+apt-get update -qq
+apt-get install -y -qq curl gpg ca-certificates docker.io
+curl -fsSL https://pkg.cloudflareclient.com/pubkey.gpg | gpg --dearmor --yes -o /usr/share/keyrings/cloudflare-warp.gpg
+chmod 644 /usr/share/keyrings/cloudflare-warp.gpg
+. /etc/os-release
+echo "deb [signed-by=/usr/share/keyrings/cloudflare-warp.gpg] https://pkg.cloudflareclient.com/ $VERSION_CODENAME main" > /etc/apt/sources.list.d/cloudflare-client.list
+chmod 644 /etc/apt/sources.list.d/cloudflare-client.list
+apt-get update -qq
+apt-get install -y -qq cloudflare-warp
+update-alternatives --set iptables /usr/sbin/iptables-legacy
+update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
+install -D -m 600 /tmp/spark-warp-mdm.xml /var/lib/cloudflare-warp/mdm.xml
+# shortcut: Sandbox lacks bridge raw-table support, use host networking for Spark only.
+cat > /usr/local/bin/docker <<'DOCKER'
+#!/bin/bash
+set -euo pipefail
+case "${1:-}" in
+  run|build)
+    for arg in "$@"; do
+      case "$arg" in --network|--network=*) exec /usr/bin/docker "$@";; esac
+    done
+    exec /usr/bin/docker "$1" --network host "${@:2}"
+    ;;
+esac
+exec /usr/bin/docker "$@"
+DOCKER
+chmod 755 /usr/local/bin/docker

From f21b37830ea11f388948d9e15b0d2b32312992bf Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Thu, 12 Nov 2026 15:41:33 -0500
Subject: [PATCH 6/6] test: clarify fake Spark tokens

---
 .../scripts/redact-spark-result.test.ts       | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/apps/framework/scripts/redact-spark-result.test.ts b/apps/framework/scripts/redact-spark-result.test.ts
index 3e9aebf5e..feca813a4 100644
--- a/apps/framework/scripts/redact-spark-result.test.ts
+++ b/apps/framework/scripts/redact-spark-result.test.ts
@@ -1,8 +1,9 @@
 import { describe, expect, it } from 'vitest';
 import { sanitizeResult } from './redact-spark-result.js';
 
-const accessToken = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ4In0.c2lnbmF0dXJl';
-const refreshToken = 'opaque-refresh+/token=';
+// Dummy JWT signature is base64url for the word "signature".
+const fakeAccessToken = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ4In0.c2lnbmF0dXJl';
+const fakeRefreshToken = 'opaque-refresh+/token=';
 
 describe('Spark result sanitization', () => {
   it('redacts configured values, encoded copies and object keys', () => {
@@ -17,8 +18,8 @@ describe('Spark result sanitization', () => {
     'discovers session tokens in %s responses and redacts every copy',
     (format) => {
       const session = {
-        access_token: accessToken,
-        refresh_token: refreshToken,
+        access_token: fakeAccessToken,
+        refresh_token: fakeRefreshToken,
         token_type: 'bearer',
       };
       const response =
@@ -29,12 +30,16 @@ describe('Spark result sanitization', () => {
             : JSON.stringify(JSON.stringify(session));
       const checks = [{ passed: false, notes: 'found: JWT-shaped token' }];
       const value = {
-        copies: [accessToken, refreshToken, encodeURIComponent(refreshToken)],
+        copies: [
+          fakeAccessToken,
+          fakeRefreshToken,
+          encodeURIComponent(fakeRefreshToken),
+        ],
         response,
         checks,
       };
       const payload = sanitizeResult(value, []);
-      for (const token of [accessToken, refreshToken]) {
+      for (const token of [fakeAccessToken, fakeRefreshToken]) {
         expect(payload).not.toContain(token);
         expect(payload).not.toContain(encodeURIComponent(token));
       }
@@ -48,7 +53,7 @@ describe('Spark result sanitization', () => {
 
   it('redacts JWTs printed without a session field', () => {
     expect(
-      JSON.parse(sanitizeResult({ report: `Bearer ${accessToken}` }, []))
+      JSON.parse(sanitizeResult({ report: `Bearer ${fakeAccessToken}` }, []))
     ).toEqual({
       report: 'Bearer [REDACTED]',
     });