feat: enable Spark evals in CI

.github/workflows/eval-refresh.yml

@@ -320,6 +320,9 @@ jobs:
       VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
       VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }}
       VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
+      SPARK_API_URL: ${{ secrets.SPARK_API_URL }}
+      CLOUDFLARE_WARP_CLIENT_ID: ${{ secrets.CLOUDFLARE_WARP_CLIENT_ID }}
+      CLOUDFLARE_WARP_CLIENT_SECRET: ${{ secrets.CLOUDFLARE_WARP_CLIENT_SECRET }}
       SUPABASE_CLI_STABLE_VERSION: ${{ needs.prepare.outputs.cli_stable_version }}
       SUPABASE_CLI_BETA_VERSION: ${{ needs.prepare.outputs.cli_beta_version }}
       SUPABASE_CLI_NEXT_VERSION: ${{ needs.prepare.outputs.cli_next_version }}

CONTRIBUTING.md

@@ -46,6 +46,10 @@ Add a `*.experiment.ts` file under `experiments/<suite>/` for the agent, model,
 
 The suite folder decides which evals the experiment runs: `experiments/<suite>/` runs the evals in `evals/<suite>/`. If this experiment should be part of our published benchmark, put it in `experiments/benchmark/` and include a corresponding `*-no-skills` variant (`skills: []`) to compare results with and without skills. To run one experiment in several suites, export a shared preset from `experiments/presets.ts` and spread it in a file under each suite folder.
 
+## Spark evals
+
+Local runs need `SPARK_API_URL` and WARP access. For CI, configure `SPARK_API_URL`, `CLOUDFLARE_WARP_CLIENT_ID`, and `CLOUDFLARE_WARP_CLIENT_SECRET` as repository secrets alongside the existing Vercel and provider credentials.
+
 ## Submitting evals for review
 
 Before submitting an eval for review, try running it locally to sanity check that it can complete without errors. It's okay if agents fail the eval, we just don't want them to be scored unfairly for framework limitations.

apps/framework/harness/run-eval.ts

@@ -530,6 +530,7 @@ async function runOne(
     ? disposable(
         await createBareSandbox({
           env: session.env,
+          mountDockerSocket: session.mountDockerSocket,
           skills: skillSources,
           mounts: supabaseMcpServerMounts(),
         })

apps/framework/scripts/redact-spark-result.test.ts

@@ -0,0 +1,61 @@
+import { describe, expect, it } from 'vitest';
+import { sanitizeResult } from './redact-spark-result.js';
+
+// Dummy JWT signature is base64url for the word "signature".
+const fakeAccessToken = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ4In0.c2lnbmF0dXJl';
+const fakeRefreshToken = 'opaque-refresh+/token=';
+
+describe('Spark result sanitization', () => {
+  it('redacts configured values, encoded copies and object keys', () => {
+    const secret = 'https://private.example/a?token=secret';
+    const value = { [secret]: [secret, encodeURIComponent(secret), 4] };
+    expect(JSON.parse(sanitizeResult(value, [secret]))).toEqual({
+      '[REDACTED]': ['[REDACTED]', '[REDACTED]', 4],
+    });
+  });
+
+  it.each(['structured', 'stdout', 'escaped'])(
+    'discovers session tokens in %s responses and redacts every copy',
+    (format) => {
+      const session = {
+        access_token: fakeAccessToken,
+        refresh_token: fakeRefreshToken,
+        token_type: 'bearer',
+      };
+      const response =
+        format === 'structured'
+          ? session
+          : format === 'stdout'
+            ? `signup response: ${JSON.stringify(session)}`
+            : JSON.stringify(JSON.stringify(session));
+      const checks = [{ passed: false, notes: 'found: JWT-shaped token' }];
+      const value = {
+        copies: [
+          fakeAccessToken,
+          fakeRefreshToken,
+          encodeURIComponent(fakeRefreshToken),
+        ],
+        response,
+        checks,
+      };
+      const payload = sanitizeResult(value, []);
+      for (const token of [fakeAccessToken, fakeRefreshToken]) {
+        expect(payload).not.toContain(token);
+        expect(payload).not.toContain(encodeURIComponent(token));
+      }
+      expect(JSON.parse(payload)).toMatchObject({
+        copies: ['[REDACTED]', '[REDACTED]', '[REDACTED]'],
+        checks,
+      });
+      expect(payload).toContain('bearer');
+    }
+  );
+
+  it('redacts JWTs printed without a session field', () => {
+    expect(
+      JSON.parse(sanitizeResult({ report: `Bearer ${fakeAccessToken}` }, []))
+    ).toEqual({
+      report: 'Bearer [REDACTED]',
+    });
+  });
+});

apps/framework/scripts/redact-spark-result.ts

@@ -0,0 +1,105 @@
+import { readFileSync, writeFileSync } from 'node:fs';
+
+const JWT = /eyJ[\w-]+\.[\w-]+\.[\w-]+/g;
+const SESSION_TOKEN =
+  /\\*["'](?:access_token|refresh_token|id_token)\\*["']\s*:\s*\\*["']([\w.~+/%=-]+)\\*["']/g;
+
+function sessionTokens(value: unknown): string[] {
+  if (typeof value === 'string') {
+    return [...value.matchAll(JWT)]
+      .map((match) => match[0])
+      .concat([...value.matchAll(SESSION_TOKEN)].map((match) => match[1]));
+  }
+  if (Array.isArray(value)) return value.flatMap(sessionTokens);
+  if (value && typeof value === 'object') {
+    return Object.entries(value).flatMap(([key, item]) => [
+      ...(['access_token', 'refresh_token', 'id_token'].includes(key) &&
+      typeof item === 'string'
+        ? [item]
+        : []),
+      ...sessionTokens(key),
+      ...sessionTokens(item),
+    ]);
+  }
+  return [];
+}
+
+function encodedSecret(secret: string): string {
+  return encodeURIComponent(secret).replace(
+    /[!'()*]/g,
+    (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`
+  );
+}
+
+function redact(value: unknown, secrets: readonly string[]): unknown {
+  if (typeof value === 'string') {
+    let text = value;
+    for (const secret of secrets) {
+      text = text
+        .replaceAll(secret, '[REDACTED]')
+        .replaceAll(encodedSecret(secret), '[REDACTED]');
+    }
+    return text;
+  }
+  if (Array.isArray(value)) return value.map((item) => redact(item, secrets));
+  if (value && typeof value === 'object') {
+    return Object.fromEntries(
+      Object.entries(value).map(([key, item]) => [
+        String(redact(key, secrets)),
+        redact(item, secrets),
+      ])
+    );
+  }
+  return value;
+}
+
+export function sanitizeResult(
+  value: unknown,
+  configuredSecrets: readonly string[]
+): string {
+  const secrets = [...new Set([...configuredSecrets, ...sessionTokens(value)])]
+    .filter(Boolean)
+    .sort((a, b) => b.length - a.length);
+  const payload = JSON.stringify(redact(value, secrets));
+  if (
+    secrets.some(
+      (secret) =>
+        payload.includes(secret) || payload.includes(encodedSecret(secret))
+    )
+  ) {
+    throw new Error('Result still contains a private value');
+  }
+  return payload;
+}
+
+if (import.meta.main) {
+  try {
+    const secrets = [
+      '/tmp/spark-warp-secrets.json',
+      '/tmp/spark-redactions.json',
+    ].flatMap((path) => {
+      const value: unknown = JSON.parse(readFileSync(path, 'utf8'));
+      if (
+        !Array.isArray(value) ||
+        !value.every((item): item is string => typeof item === 'string')
+      )
+        throw new Error('Invalid Spark redactions');
+      return value;
+    });
+    const path = process.argv[2];
+    const value: unknown = JSON.parse(readFileSync(path, 'utf8'));
+    writeFileSync(path, sanitizeResult(value, secrets));
+  } catch (error) {
+    writeFileSync(
+      '/tmp/spark-redaction-error.log',
+      error instanceof Error
+        ? (error.stack ?? error.message)
+        : 'Redaction failed',
+      { mode: 0o600 }
+    );
+    console.error(
+      'Spark result redaction failed. Inspect private Sandbox logs.'
+    );
+    process.exitCode = 1;
+  }
+}

apps/framework/scripts/run-vercel-evals.test.ts

@@ -1,4 +1,5 @@
-import { APIError } from '@vercel/sandbox';
+import { APIError, Sandbox } from '@vercel/sandbox';
+import { AbortError } from 'p-retry';
 import { resolveCliVersion, type CliChannel } from '@supabase-evals/sandbox';
 import { execFileSync } from 'node:child_process';
 import {
@@ -29,15 +30,76 @@ import {
   requiredCliChannels,
   resolveChannelPins,
   runBounded,
+  runPairOnce,
   tagValue,
   expandJobs,
   type EvalPair,
+  sparkWarpMdm,
 } from './run-vercel-evals.js';
 
 vi.mock('@supabase-evals/sandbox', () => ({ resolveCliVersion: vi.fn() }));
 
 const BROKERED_NAMES = BROKERED_KEYS.map(({ name }) => name);
 
+it('escapes credentials as XML text in the WARP configuration', () => {
+  expect(sparkWarpMdm('client<&>', 'secret<&>')).toContain(
+    '<key>auth_client_id</key><string>client&lt;&amp;&gt;</string>'
+  );
+  expect(sparkWarpMdm('client<&>', 'secret<&>')).toContain(
+    '<key>auth_client_secret</key><string>secret&lt;&amp;&gt;</string>'
+  );
+});
+
+describe('Spark terminal failures', () => {
+  const options = {
+    pair: {
+      eval_id: 'eval-1',
+      experiment: 'experiment-1',
+      eval_suite: 'spark',
+    },
+    pairs: [],
+    revision: 'test',
+    repoUrl: 'https://github.com/supabase/evals',
+    outputDir: '/tmp/unused-spark-results',
+    runs: 1,
+    timeoutSec: 1,
+    concurrency: 1,
+    vcpus: 1,
+    attempt: 1,
+    run: 1,
+    pins: {},
+  };
+  const credentials = { token: 'test', teamId: 'test', projectId: 'test' };
+
+  beforeEach(() => {
+    for (const name of BROKERED_NAMES) vi.stubEnv(name, '');
+    vi.stubEnv('SPARK_API_URL', 'https://control.example');
+    vi.stubEnv('CLOUDFLARE_WARP_CLIENT_ID', 'test.access');
+    vi.stubEnv('CLOUDFLARE_WARP_CLIENT_SECRET', 'test');
+  });
+  afterEach(() => {
+    vi.unstubAllEnvs();
+    vi.restoreAllMocks();
+  });
+
+  it.each(['unauthorized Sandbox', 'missing credentials'])(
+    'aborts %s without retrying',
+    async (reason) => {
+      if (reason === 'missing credentials')
+        vi.stubEnv('CLOUDFLARE_WARP_CLIENT_SECRET', '');
+      const create = vi
+        .spyOn(Sandbox, 'create')
+        .mockRejectedValue(new APIError(new Response(null, { status: 401 })));
+      await expect(runPairOnce(options, credentials)).rejects.toBeInstanceOf(
+        AbortError
+      );
+      expect(create).toHaveBeenCalledTimes(
+        reason === 'missing credentials' ? 0 : 1
+      );
+    }
+  );
+});
+
 describe('agentEnvironment', () => {
   const originalValues = new Map<string, string | undefined>();
 
@@ -422,6 +484,27 @@ describe('Vercel eval controller', () => {
     }
   });
 
+  it('downloads only sanitized JSON for Spark, never workspace or session archives', async () => {
+    const downloadFile = vi.fn().mockResolvedValue('downloaded');
+    const result = await downloadResults(
+      { downloadFile },
+      { eval_id: 'eval-1', experiment: 'experiment-1', eval_suite: 'spark' },
+      1,
+      '/tmp/unused-spark-results',
+      '/vercel/evals'
+    );
+    expect(downloadFile).toHaveBeenCalledExactlyOnceWith(
+      {
+        path: '/vercel/evals/results/spark/experiment-1/eval-1/run-1/result.json',
+      },
+      { path: result.partialPath },
+      { mkdirRecursive: true }
+    );
+    expect(result.partialPath).toBe(
+      '/tmp/unused-spark-results/raw-results-spark__experiment-1__eval-1/eval-1/run-1/result.json.partial'
+    );
+  });
+
   it('downloads result metadata separately from agent workspace files', async () => {
     const temporary = mkdtempSync(join(tmpdir(), 'vercel-eval-test-'));
     const sandboxFiles = join(temporary, 'sandbox');