diff --git a/packages/sandbox/Dockerfile b/packages/sandbox/Dockerfile index 4885ae3f..a27c18cf 100644 --- a/packages/sandbox/Dockerfile +++ b/packages/sandbox/Dockerfile @@ -5,9 +5,16 @@ FROM node:22-slim RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \ - ca-certificates curl git docker.io postgresql-client \ + ca-certificates curl git postgresql-client \ && rm -rf /var/lib/apt/lists/* +# The Docker CLI (talking to the host daemon through the mounted socket), from +# Docker's official static build rather than Debian's package, which ships +# 20.10. With a client older than 23.0, `supabase stop --no-backup` +# on CLI versions before 2.122.0 removes the containers but fails to prune the +# database volume (CLI-2637), so the next run would boot from it. +COPY --from=docker:27.5.1-cli /usr/local/bin/docker /usr/local/bin/docker + # Global npm installs (both the baked-in skills CLI below and the agent's own # `npm install -g supabase` at setup time) land in a node-owned prefix instead # of /usr/local, so they work without root/sudo — matching a typical user's diff --git a/packages/sandbox/src/agent-environment.ts b/packages/sandbox/src/agent-environment.ts index 52b146a7..4fe5aba2 100644 --- a/packages/sandbox/src/agent-environment.ts +++ b/packages/sandbox/src/agent-environment.ts @@ -18,6 +18,7 @@ import { DockerSandbox } from './docker-sandbox.js'; import { ensureSupabaseSandboxImage, setupSupabaseSandbox, + teardownSupabaseProject, } from './supabase.js'; import { installSkills, type SkillEntry } from './skills.js'; @@ -100,6 +101,9 @@ export async function createAgentEnvironment( const skills = await installSkills(sandbox, options.skills ?? []); return { sandbox, skills, close: () => sandbox.stop() }; } catch (err) { + // A failed setup (e.g. `supabase start` timing out) can leave a stack + // running; stopping the sandbox alone would leak it into the next run. + if (options.localStack) await teardownSupabaseProject(sandbox); await sandbox.stop(); throw err; } diff --git a/packages/sandbox/src/supabase.ts b/packages/sandbox/src/supabase.ts index 8c200de3..c82feaad 100644 --- a/packages/sandbox/src/supabase.ts +++ b/packages/sandbox/src/supabase.ts @@ -14,6 +14,7 @@ * needed. */ +import { createHash } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { @@ -103,14 +104,20 @@ export const SANDBOX_DOCKERFILE_PATH = fileURLToPath( * the layer cache makes repeat calls effectively free. */ export async function ensureSupabaseSandboxImage(): Promise { - // The base image carries only common tooling; its sole version input is the - // skills CLI. The Supabase CLI is installed per local-stack session - // (installSupabaseCli), so the image is shared across modes and CLI versions. - const tag = `${SANDBOX_IMAGE_REPOSITORY}:base-skills-${SKILLS_CLI_VERSION}`; + // The base image carries only common tooling; its inputs are the skills CLI + // version and the Dockerfile itself, hashed so an edited Dockerfile gets a + // new tag instead of reusing a stale cached image. The Supabase CLI is + // installed per local-stack session (installSupabaseCli), so the image is + // shared across modes and CLI versions. + const dockerfile = readFileSync(SANDBOX_DOCKERFILE_PATH, 'utf8'); + const dockerfileHash = createHash('sha256') + .update(dockerfile) + .digest('hex') + .slice(0, 12); + const tag = `${SANDBOX_IMAGE_REPOSITORY}:base-skills-${SKILLS_CLI_VERSION}-${dockerfileHash}`; const existing = await dockerCli(['image', 'inspect', tag]); if (existing.ok) return tag; - const dockerfile = readFileSync(SANDBOX_DOCKERFILE_PATH, 'utf8'); await withTransientRetries( `build ${tag}`, () => @@ -258,6 +265,17 @@ export async function setupSupabaseSandbox( await sandbox.copyToContainer(options.localDir, sandbox.workdir); } + // A run that died before teardown (Ctrl-C, crash) leaves its stack behind + // under this workspace's project id, and the next `supabase start` would + // boot from its database. Remove it first, scoped to this project. After the + // hosted link, whose SUPABASE_PROJECT_ID names the stack. + if ( + !options.skipCliInstall && + (await sandbox.fileExists('supabase/config.toml')) + ) { + await stopSupabaseProject(sandbox); + } + // When linked to a hosted project with a wire endpoint, the CLI wrapper routes // linked DB commands (`db push`/`migration repair`/…) at it via --db-url (the // CLI otherwise hardcodes the linked port to 5432). This is needed whether or @@ -478,17 +496,32 @@ export async function ensureEdgeRuntime( export async function teardownSupabaseProject( sandbox: DockerSandbox ): Promise { + await stopSupabaseProject(sandbox); + await cleanupEvalSupabaseResources(sandbox); +} + +/** + * Stop this workspace's local stack and delete its volumes. Scoped to the + * workspace's project (no --all: a developer's own unrelated local stacks must + * survive eval runs). Best-effort: never throws, but logs a failure, which + * would otherwise leave the database volume for the next run to boot from. + */ +async function stopSupabaseProject(sandbox: DockerSandbox): Promise { try { - // Scoped stop for this workspace's project (no --all: a developer's own - // unrelated local stacks must survive eval runs). - await sandbox.runShell('supabase stop --no-backup', { timeoutMs: 120_000 }); + const stop = await sandbox.runShell('supabase stop --no-backup', { + timeoutMs: 120_000, + }); + if (!stop.ok) { + console.warn( + `[sandbox] supabase stop --no-backup exited ${stop.exitCode}: ${(stop.stderr || stop.stdout).trim()}` + ); + } } catch (err) { console.warn( - '[sandbox] supabase stop failed (continuing with docker cleanup):', + '[sandbox] supabase stop --no-backup failed:', err instanceof Error ? err.message : err ); } - await cleanupEvalSupabaseResources(sandbox); } /** diff --git a/packages/sandbox/test/docker.test.ts b/packages/sandbox/test/docker.test.ts index ef68bca2..da3eef61 100644 --- a/packages/sandbox/test/docker.test.ts +++ b/packages/sandbox/test/docker.test.ts @@ -5,15 +5,17 @@ * * pnpm --filter @supabase-evals/sandbox test:docker */ +import { randomUUID } from 'node:crypto'; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterAll, describe, expect, it } from 'vitest'; +import { createAgentEnvironment } from '../src/agent-environment.js'; import { buildLocalStackScoringContext, localStackRuntime, } from '../src/local-stack-runtime.js'; -import { DockerSandbox } from '../src/docker-sandbox.js'; +import { DockerSandbox, dockerCli } from '../src/docker-sandbox.js'; import { ensureSupabaseSandboxImage, setupSupabaseSandbox, @@ -342,5 +344,132 @@ describe.runIf(process.env.SANDBOX_DOCKER_TESTS)( } } ); + + describe('each session starts from a fresh database', () => { + // A fixture project id without the `sandbox-` prefix, like most eval + // fixtures, on its own ports so it can't collide with other stacks. + const projectId = `evals-fresh-db-${randomUUID().slice(0, 8)}`; + const label = `label=com.supabase.cli.project=${projectId}`; + const fixture = mkdtempSync(join(tmpdir(), 'fresh-db-fixture-')); + mkdirSync(join(fixture, 'supabase', 'migrations'), { recursive: true }); + writeFileSync( + join(fixture, 'supabase', 'config.toml'), + [ + `project_id = "${projectId}"`, + '[api]', + 'port = 55621', + '[db]', + 'port = 55622', + 'shadow_port = 55620', + '[studio]', + 'port = 55623', + '[inbucket]', + 'port = 55624', + '[analytics]', + 'port = 55627', + '', + ].join('\n') + ); + writeFileSync( + join(fixture, 'supabase', 'migrations', '20260101000000_seed.sql'), + 'create table public.seeded (id int);\n' + ); + const agentMigration = '20260102000000'; + + async function listed(kind: 'ps' | 'volume'): Promise { + const args = kind === 'ps' ? ['ps', '-aq'] : ['volume', 'ls', '-q']; + const result = await dockerCli([...args, '--filter', label]); + return result.stdout.split('\n').filter(Boolean); + } + + afterAll(async () => { + const containers = await listed('ps'); + if (containers.length) await dockerCli(['rm', '-f', ...containers]); + const volumes = await listed('volume'); + if (volumes.length) await dockerCli(['volume', 'rm', ...volumes]); + const networks = await dockerCli([ + 'network', + 'ls', + '-q', + '--filter', + label, + ]); + const ids = networks.stdout.split('\n').filter(Boolean); + if (ids.length) await dockerCli(['network', 'rm', ...ids]); + rmSync(fixture, { recursive: true, force: true }); + }); + + /** Boot a session on the fixture, as the local-stack runtime does. */ + async function startRun() { + const env = await createAgentEnvironment({ + localDir: fixture, + localStack: { includeServices: [] }, + }); + return { + sandbox: env.sandbox, + close: async () => { + await teardownSupabaseProject(env.sandbox); + await env.close(); + }, + }; + } + + /** What an agent leaves behind: an applied migration and a new table. */ + async function changeDatabase(sandbox: DockerSandbox): Promise { + await sandbox.writeFiles({ + [`supabase/migrations/${agentMigration}_agent.sql`]: + 'create table public.agent_change (id int);\n', + }); + const up = await sandbox.runShell('supabase migration up --local'); + expect(up.ok, up.stderr).toBe(true); + } + + async function expectFreshDatabase( + sandbox: DockerSandbox + ): Promise { + const { rows } = await buildLocalStackScoringContext(sandbox).query( + "select to_regclass('public.seeded')::text as seeded, to_regclass('public.agent_change')::text as agent_change" + ); + expect(rows).toEqual([{ seeded: 'seeded', agent_change: null }]); + const history = await sandbox.runShell( + 'supabase migration list --local' + ); + expect(history.ok, history.stderr).toBe(true); + expect(history.stdout).toContain('20260101000000'); + expect(history.stdout).not.toContain(agentMigration); + } + + it( + 'after a torn-down run and after a run killed before teardown', + { timeout: TEST_TIMEOUT_MS }, + async () => { + const first = await startRun(); + try { + await changeDatabase(first.sandbox); + } finally { + await first.close(); + } + // Teardown deletes the database volume, not just the containers. + expect(await listed('volume')).toEqual([]); + + const second = await startRun(); + await expectFreshDatabase(second.sandbox); + await changeDatabase(second.sandbox); + // Killed before teardown (e.g. Ctrl-C): the stack leaks... + await second.sandbox.stop(); + expect(await listed('volume')).not.toEqual([]); + + // ...and the next run removes it before starting. + const third = await startRun(); + try { + await expectFreshDatabase(third.sandbox); + } finally { + await third.close(); + } + expect(await listed('volume')).toEqual([]); + expect(await listed('ps')).toEqual([]); + } + ); + }); } ); diff --git a/packages/sandbox/test/unit.test.ts b/packages/sandbox/test/unit.test.ts index 50db7dec..91083550 100644 --- a/packages/sandbox/test/unit.test.ts +++ b/packages/sandbox/test/unit.test.ts @@ -40,7 +40,12 @@ describe('sandbox Dockerfile', () => { expect(dockerfile).toContain('FROM node:22-slim'); // Common tooling shared by both eval modes. expect(dockerfile).toContain('postgresql-client'); - expect(dockerfile).toContain('docker.io'); + // Docker's own CLI, not Debian's docker.io (20.10): older than 23.0, it makes + // `supabase stop --no-backup` leave the database volume behind (CLI-2637). + expect(dockerfile).toMatch( + /COPY --from=docker:2[3-9][.\d]*-cli \/usr\/local\/bin\/docker / + ); + expect(dockerfile).not.toContain('docker.io'); // The Supabase CLI is NOT baked in — it's a local-stack component installed // at setup time (installSupabaseCli), so tools-mode sandboxes genuinely lack // it. The base image is therefore shared across modes and CLI versions.