fix(sandbox): start each local eval run on a fresh database

packages/sandbox/Dockerfile

@@ -5,9 +5,16 @@
 FROM node:22-slim
 
 RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \
-    ca-certificates curl git docker.io postgresql-client \
+    ca-certificates curl git postgresql-client \
   && rm -rf /var/lib/apt/lists/*
 
+# The Docker CLI (talking to the host daemon through the mounted socket), from
+# Docker's official static build rather than Debian's package, which ships
+# 20.10. With a client older than 23.0, `supabase stop --no-backup`
+# on CLI versions before 2.122.0 removes the containers but fails to prune the
+# database volume (CLI-2637), so the next run would boot from it.
+COPY --from=docker:27.5.1-cli /usr/local/bin/docker /usr/local/bin/docker
+
 # Global npm installs (both the baked-in skills CLI below and the agent's own
 # `npm install -g supabase` at setup time) land in a node-owned prefix instead
 # of /usr/local, so they work without root/sudo — matching a typical user's

packages/sandbox/src/agent-environment.ts

@@ -18,6 +18,7 @@ import { DockerSandbox } from './docker-sandbox.js';
 import {
   ensureSupabaseSandboxImage,
   setupSupabaseSandbox,
+  teardownSupabaseProject,
 } from './supabase.js';
 import { installSkills, type SkillEntry } from './skills.js';
 
@@ -100,6 +101,9 @@ export async function createAgentEnvironment(
     const skills = await installSkills(sandbox, options.skills ?? []);
     return { sandbox, skills, close: () => sandbox.stop() };
   } catch (err) {
+    // A failed setup (e.g. `supabase start` timing out) can leave a stack
+    // running; stopping the sandbox alone would leak it into the next run.
+    if (options.localStack) await teardownSupabaseProject(sandbox);
     await sandbox.stop();
     throw err;
   }

packages/sandbox/src/supabase.ts

@@ -14,6 +14,7 @@
  * needed.
  */
 
+import { createHash } from 'node:crypto';
 import { readFileSync } from 'node:fs';
 import { fileURLToPath } from 'node:url';
 import {
@@ -103,14 +104,20 @@ export const SANDBOX_DOCKERFILE_PATH = fileURLToPath(
  * the layer cache makes repeat calls effectively free.
  */
 export async function ensureSupabaseSandboxImage(): Promise<string> {
-  // The base image carries only common tooling; its sole version input is the
-  // skills CLI. The Supabase CLI is installed per local-stack session
-  // (installSupabaseCli), so the image is shared across modes and CLI versions.
-  const tag = `${SANDBOX_IMAGE_REPOSITORY}:base-skills-${SKILLS_CLI_VERSION}`;
+  // The base image carries only common tooling; its inputs are the skills CLI
+  // version and the Dockerfile itself, hashed so an edited Dockerfile gets a
+  // new tag instead of reusing a stale cached image. The Supabase CLI is
+  // installed per local-stack session (installSupabaseCli), so the image is
+  // shared across modes and CLI versions.
+  const dockerfile = readFileSync(SANDBOX_DOCKERFILE_PATH, 'utf8');
+  const dockerfileHash = createHash('sha256')
+    .update(dockerfile)
+    .digest('hex')
+    .slice(0, 12);
+  const tag = `${SANDBOX_IMAGE_REPOSITORY}:base-skills-${SKILLS_CLI_VERSION}-${dockerfileHash}`;
   const existing = await dockerCli(['image', 'inspect', tag]);
   if (existing.ok) return tag;
 
-  const dockerfile = readFileSync(SANDBOX_DOCKERFILE_PATH, 'utf8');
   await withTransientRetries(
     `build ${tag}`,
     () =>
@@ -258,6 +265,17 @@ export async function setupSupabaseSandbox(
     await sandbox.copyToContainer(options.localDir, sandbox.workdir);
   }
 
+  // A run that died before teardown (Ctrl-C, crash) leaves its stack behind
+  // under this workspace's project id, and the next `supabase start` would
+  // boot from its database. Remove it first, scoped to this project. After the
+  // hosted link, whose SUPABASE_PROJECT_ID names the stack.
+  if (
+    !options.skipCliInstall &&
+    (await sandbox.fileExists('supabase/config.toml'))
+  ) {
+    await stopSupabaseProject(sandbox);
+  }
+
   // When linked to a hosted project with a wire endpoint, the CLI wrapper routes
   // linked DB commands (`db push`/`migration repair`/…) at it via --db-url (the
   // CLI otherwise hardcodes the linked port to 5432). This is needed whether or
@@ -478,17 +496,32 @@ export async function ensureEdgeRuntime(
 export async function teardownSupabaseProject(
   sandbox: DockerSandbox
 ): Promise<void> {
+  await stopSupabaseProject(sandbox);
+  await cleanupEvalSupabaseResources(sandbox);
+}
+
+/**
+ * Stop this workspace's local stack and delete its volumes. Scoped to the
+ * workspace's project (no --all: a developer's own unrelated local stacks must
+ * survive eval runs). Best-effort: never throws, but logs a failure, which
+ * would otherwise leave the database volume for the next run to boot from.
+ */
+async function stopSupabaseProject(sandbox: DockerSandbox): Promise<void> {
   try {
-    // Scoped stop for this workspace's project (no --all: a developer's own
-    // unrelated local stacks must survive eval runs).
-    await sandbox.runShell('supabase stop --no-backup', { timeoutMs: 120_000 });
+    const stop = await sandbox.runShell('supabase stop --no-backup', {
+      timeoutMs: 120_000,
+    });
+    if (!stop.ok) {
+      console.warn(
+        `[sandbox] supabase stop --no-backup exited ${stop.exitCode}: ${(stop.stderr || stop.stdout).trim()}`
+      );
+    }
   } catch (err) {
     console.warn(
-      '[sandbox] supabase stop failed (continuing with docker cleanup):',
+      '[sandbox] supabase stop --no-backup failed:',
       err instanceof Error ? err.message : err
     );
   }
-  await cleanupEvalSupabaseResources(sandbox);
 }
 
 /**

packages/sandbox/test/docker.test.ts

@@ -5,15 +5,17 @@
  *
  *   pnpm --filter @supabase-evals/sandbox test:docker
  */
+import { randomUUID } from 'node:crypto';
 import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { afterAll, describe, expect, it } from 'vitest';
+import { createAgentEnvironment } from '../src/agent-environment.js';
 import {
   buildLocalStackScoringContext,
   localStackRuntime,
 } from '../src/local-stack-runtime.js';
-import { DockerSandbox } from '../src/docker-sandbox.js';
+import { DockerSandbox, dockerCli } from '../src/docker-sandbox.js';
 import {
   ensureSupabaseSandboxImage,
   setupSupabaseSandbox,
@@ -342,5 +344,132 @@ describe.runIf(process.env.SANDBOX_DOCKER_TESTS)(
         }
       }
     );
+
+    describe('each session starts from a fresh database', () => {
+      // A fixture project id without the `sandbox-` prefix, like most eval
+      // fixtures, on its own ports so it can't collide with other stacks.
+      const projectId = `evals-fresh-db-${randomUUID().slice(0, 8)}`;
+      const label = `label=com.supabase.cli.project=${projectId}`;
+      const fixture = mkdtempSync(join(tmpdir(), 'fresh-db-fixture-'));
+      mkdirSync(join(fixture, 'supabase', 'migrations'), { recursive: true });
+      writeFileSync(
+        join(fixture, 'supabase', 'config.toml'),
+        [
+          `project_id = "${projectId}"`,
+          '[api]',
+          'port = 55621',
+          '[db]',
+          'port = 55622',
+          'shadow_port = 55620',
+          '[studio]',
+          'port = 55623',
+          '[inbucket]',
+          'port = 55624',
+          '[analytics]',
+          'port = 55627',
+          '',
+        ].join('\n')
+      );
+      writeFileSync(
+        join(fixture, 'supabase', 'migrations', '20260101000000_seed.sql'),
+        'create table public.seeded (id int);\n'
+      );
+      const agentMigration = '20260102000000';
+
+      async function listed(kind: 'ps' | 'volume'): Promise<string[]> {
+        const args = kind === 'ps' ? ['ps', '-aq'] : ['volume', 'ls', '-q'];
+        const result = await dockerCli([...args, '--filter', label]);
+        return result.stdout.split('\n').filter(Boolean);
+      }
+
+      afterAll(async () => {
+        const containers = await listed('ps');
+        if (containers.length) await dockerCli(['rm', '-f', ...containers]);
+        const volumes = await listed('volume');
+        if (volumes.length) await dockerCli(['volume', 'rm', ...volumes]);
+        const networks = await dockerCli([
+          'network',
+          'ls',
+          '-q',
+          '--filter',
+          label,
+        ]);
+        const ids = networks.stdout.split('\n').filter(Boolean);
+        if (ids.length) await dockerCli(['network', 'rm', ...ids]);
+        rmSync(fixture, { recursive: true, force: true });
+      });
+
+      /** Boot a session on the fixture, as the local-stack runtime does. */
+      async function startRun() {
+        const env = await createAgentEnvironment({
+          localDir: fixture,
+          localStack: { includeServices: [] },
+        });
+        return {
+          sandbox: env.sandbox,
+          close: async () => {
+            await teardownSupabaseProject(env.sandbox);
+            await env.close();
+          },
+        };
+      }
+
+      /** What an agent leaves behind: an applied migration and a new table. */
+      async function changeDatabase(sandbox: DockerSandbox): Promise<void> {
+        await sandbox.writeFiles({
+          [`supabase/migrations/${agentMigration}_agent.sql`]:
+            'create table public.agent_change (id int);\n',
+        });
+        const up = await sandbox.runShell('supabase migration up --local');
+        expect(up.ok, up.stderr).toBe(true);
+      }
+
+      async function expectFreshDatabase(
+        sandbox: DockerSandbox
+      ): Promise<void> {
+        const { rows } = await buildLocalStackScoringContext(sandbox).query(
+          "select to_regclass('public.seeded')::text as seeded, to_regclass('public.agent_change')::text as agent_change"
+        );
+        expect(rows).toEqual([{ seeded: 'seeded', agent_change: null }]);
+        const history = await sandbox.runShell(
+          'supabase migration list --local'
+        );
+        expect(history.ok, history.stderr).toBe(true);
+        expect(history.stdout).toContain('20260101000000');
+        expect(history.stdout).not.toContain(agentMigration);
+      }
+
+      it(
+        'after a torn-down run and after a run killed before teardown',
+        { timeout: TEST_TIMEOUT_MS },
+        async () => {
+          const first = await startRun();
+          try {
+            await changeDatabase(first.sandbox);
+          } finally {
+            await first.close();
+          }
+          // Teardown deletes the database volume, not just the containers.
+          expect(await listed('volume')).toEqual([]);
+
+          const second = await startRun();
+          await expectFreshDatabase(second.sandbox);
+          await changeDatabase(second.sandbox);
+          // Killed before teardown (e.g. Ctrl-C): the stack leaks...
+          await second.sandbox.stop();
+          expect(await listed('volume')).not.toEqual([]);
+
+          // ...and the next run removes it before starting.
+          const third = await startRun();
+          try {
+            await expectFreshDatabase(third.sandbox);
+          } finally {
+            await third.close();
+          }
+          expect(await listed('volume')).toEqual([]);
+          expect(await listed('ps')).toEqual([]);
+        }
+      );
+    });
   }
 );

packages/sandbox/test/unit.test.ts

@@ -40,7 +40,12 @@ describe('sandbox Dockerfile', () => {
     expect(dockerfile).toContain('FROM node:22-slim');
     // Common tooling shared by both eval modes.
     expect(dockerfile).toContain('postgresql-client');
-    expect(dockerfile).toContain('docker.io');
+    // Docker's own CLI, not Debian's docker.io (20.10): older than 23.0, it makes
+    // `supabase stop --no-backup` leave the database volume behind (CLI-2637).
+    expect(dockerfile).toMatch(
+      /COPY --from=docker:2[3-9][.\d]*-cli \/usr\/local\/bin\/docker /
+    );
+    expect(dockerfile).not.toContain('docker.io');
     // The Supabase CLI is NOT baked in — it's a local-stack component installed
     // at setup time (installSupabaseCli), so tools-mode sandboxes genuinely lack
     // it. The base image is therefore shared across modes and CLI versions.