From 4e154fcf62428ca9df338ceacd1468204fd1578f Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 22 Sep 2026 08:37:43 +0200 Subject: [PATCH 01/20] perf(stack): replace snapshot archives with managed directory snapshots --- .../stack-shadow-cache.integration.test.ts | 116 +++ .../command-internal/stack-shadow-cache.ts | 78 ++ .../stack-shadow.integration.test.ts | 292 ++++++- apps/cli/src/command-internal/stack-shadow.ts | 156 +++- apps/cli/src/commands/db/diff/SIDE_EFFECTS.md | 33 +- .../db/diff/diff.stack-cache.e2e.test.ts | 142 +++ .../commands/db/dump/dump.integration.test.ts | 2 +- apps/cli/src/commands/db/pull/SIDE_EFFECTS.md | 37 +- .../db/reset/reset.integration.test.ts | 2 +- .../generate/generate.integration.test.ts | 2 +- .../schema/declarative/sync/SIDE_EFFECTS.md | 15 +- .../declarative/sync/sync.integration.test.ts | 2 +- .../db/shared/pgdelta-next-shadow.layer.ts | 14 +- ...elta-next-shadow.stack.integration.test.ts | 3 +- .../db/start/start.integration.test.ts | 2 +- .../stack/prepare/prepare.integration.test.ts | 2 +- .../stack/start/start.integration.test.ts | 2 +- .../stack/status/status.integration.test.ts | 2 +- .../serve/serve.stack.integration.test.ts | 2 +- .../commands/migration/squash/SIDE_EFFECTS.md | 16 +- apps/cli/tests/helpers/storage.ts | 2 +- packages/stack/ARCHITECTURE.md | 32 +- packages/stack/README.md | 6 +- packages/stack/src/Owner.ts | 98 +-- packages/stack/src/Rpc.ts | 11 +- packages/stack/src/StackHost.ts | 16 +- packages/stack/src/effect.ts | 13 +- packages/stack/src/index.ts | 16 +- packages/stack/src/public.e2e.test.ts | 11 +- packages/stack/src/runtime/Container.ts | 22 +- packages/stack/src/services/Catalog.ts | 37 + packages/stack/src/services/Database.ts | 184 +++- .../DatabaseSnapshot.integration.test.ts | 618 +++++++------ .../stack/src/services/DatabaseSnapshot.ts | 799 +++++++---------- .../storage/DirectoryCopy.integration.test.ts | 124 +++ packages/stack/src/storage/DirectoryCopy.ts | 179 ++++ .../DockerDatabaseStorage.integration.test.ts | 619 +++++++++++++ .../src/storage/DockerDatabaseStorage.ts | 815 ++++++++++++++++++ packages/stack/tests/whole-stack/workloads.ts | 28 +- 39 files changed, 3481 insertions(+), 1069 deletions(-) create mode 100644 apps/cli/src/command-internal/stack-shadow-cache.integration.test.ts create mode 100644 apps/cli/src/command-internal/stack-shadow-cache.ts create mode 100644 apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts create mode 100644 packages/stack/src/storage/DirectoryCopy.integration.test.ts create mode 100644 packages/stack/src/storage/DirectoryCopy.ts create mode 100644 packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts create mode 100644 packages/stack/src/storage/DockerDatabaseStorage.ts diff --git a/apps/cli/src/command-internal/stack-shadow-cache.integration.test.ts b/apps/cli/src/command-internal/stack-shadow-cache.integration.test.ts new file mode 100644 index 0000000000..d750bdea19 --- /dev/null +++ b/apps/cli/src/command-internal/stack-shadow-cache.integration.test.ts @@ -0,0 +1,116 @@ +import { CliConfigSchema } from "@supabase/config"; +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Option, Path, Schema } from "effect"; + +import type { ShadowSetupInput } from "./db-bootstrap/shadow-database.ts"; +import { stackShadowCacheEntry } from "./stack-shadow-cache.ts"; + +const defaultConfig = Schema.decodeUnknownSync(CliConfigSchema)({}); + +const input = ( + fs: FileSystem.FileSystem, + path: Path.Path, + workdir: string, + setupOverrides: Partial["setup"]> = {}, + inputOverrides: Partial, "password">> = {}, +): ShadowSetupInput => ({ + db: { major_version: 17, settings: {} }, + experimental: defaultConfig.experimental, + jwtSecret: "super-secret-jwt-token-with-at-least-32-characters-long", + jwtExpiry: 3600, + networkId: "n", + image: "stack-ephemeral", + configImage: "stack-ephemeral", + shadowPort: 54320, + password: inputOverrides.password ?? "postgres", + projectId: "proj", + isBitbucketPipeline: false, + workdir, + extraHosts: [], + fs, + path, + hostname: "127.0.0.1", + healthTimeoutSeconds: 60, + setup: { + majorVersion: 17, + config: defaultConfig, + dbUrl: "postgresql://postgres:postgres@127.0.0.1:54320/postgres", + jwtSecret: "super-secret-jwt-token-with-at-least-32-characters-long", + jwks: Effect.succeed("{}"), + apiUrl: "http://127.0.0.1:54321", + authExternalUrl: undefined, + siteUrl: "http://127.0.0.1:3000", + anonKey: "anon", + serviceRoleKey: "service", + storageTargetMigration: "", + realtimeEnabledForSetup: false, + storageEnabledForSetup: false, + authEnabledForSetup: false, + serviceVersionOverrides: {}, + projectEnvValues: { SUPABASE_SHADOW_CACHE: "1" }, + debug: false, + webhooksEnabled: false, + apiAutoExposeNewTables: Option.none(), + vault: [], + ...setupOverrides, + }, +}); + +describe("stack shadow cache entry", () => { + it.effect("rekeys roles, webhook policy, and password, and honors both cache gates", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-shadow-cache-entry-" }); + yield* fs.makeDirectory(path.join(root, "supabase"), { recursive: true }); + + const base = input(fs, path, root); + const first = yield* stackShadowCacheEntry(base, "native", "darwin", "arm64", "config"); + if (first === undefined) return yield* Effect.die("cache entry unexpectedly disabled"); + + yield* fs.writeFileString( + path.join(root, "supabase", "roles.sql"), + "CREATE ROLE cache_probe;\n", + ); + const withRoles = yield* stackShadowCacheEntry(base, "native", "darwin", "arm64", "config"); + if (withRoles === undefined) return yield* Effect.die("roles entry unexpectedly disabled"); + expect(withRoles.key).not.toBe(first.key); + + const withWebhooks = yield* stackShadowCacheEntry( + input(fs, path, root, { webhooksEnabled: true }), + "native", + "darwin", + "arm64", + "config", + ); + if (withWebhooks === undefined) + return yield* Effect.die("webhooks entry unexpectedly disabled"); + expect(withWebhooks.key).not.toBe(withRoles.key); + + const withPassword = yield* stackShadowCacheEntry( + input(fs, path, root, {}, { password: "rotated-password" }), + "native", + "darwin", + "arm64", + "config", + ); + if (withPassword === undefined) + return yield* Effect.die("password entry unexpectedly disabled"); + expect(withPassword.key).not.toBe(withRoles.key); + + expect( + yield* stackShadowCacheEntry(base, "native", "darwin", "arm64", "config", true), + ).toBeUndefined(); + expect( + yield* stackShadowCacheEntry( + input(fs, path, root, { projectEnvValues: { SUPABASE_SHADOW_CACHE: "0" } }), + "native", + "darwin", + "arm64", + "config", + ), + ).toBeUndefined(); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); +}); diff --git a/apps/cli/src/command-internal/stack-shadow-cache.ts b/apps/cli/src/command-internal/stack-shadow-cache.ts new file mode 100644 index 0000000000..7b267f4d5f --- /dev/null +++ b/apps/cli/src/command-internal/stack-shadow-cache.ts @@ -0,0 +1,78 @@ +import { Effect } from "effect"; +import { resolveArtifact, postgresVersion } from "@supabase/stack/internal/postgres-artifact"; +import type { ShadowSetupInput } from "./db-bootstrap/shadow-database.ts"; +import { + SHADOW_CACHE_ENV, + shadowCacheKey, + type ShadowCacheKeyInputs, +} from "./db-bootstrap/shadow-cache.ts"; +import { resolveSetupWebhooksEnabled, type SetupDatabaseOptions } from "./db-bootstrap/db-setup.ts"; +import { viperEnvBoolWithProjectFallback } from "./viper-env.ts"; + +interface StackShadowCacheEntry { + readonly key: string; + readonly rolesSql: string; +} + +const readRoles = (input: ShadowSetupInput) => + Effect.gen(function* () { + const file = input.path.join(input.workdir, "supabase", "roles.sql"); + if (!(yield* input.fs.exists(file))) return ""; + return yield* input.fs.readFileString(file); + }); + +export const stackShadowCacheEntry = Effect.fn("StackShadowCache.entry")(function* ( + input: ShadowSetupInput, + runtime: string, + platform: string, + arch: string, + webhooks: SetupDatabaseOptions["webhooks"], + bypassCache = false, +) { + if ( + bypassCache || + !viperEnvBoolWithProjectFallback(SHADOW_CACHE_ENV, input.setup.projectEnvValues ?? {}, { + whenUnset: true, + }) + ) + return undefined; + const rolesSql = yield* readRoles(input); + const postgres = yield* resolveArtifact({ + service: "database", + version: postgresVersion(String(input.setup.majorVersion)), + }); + const image = (service: "auth" | "storage" | "realtime", enabled: boolean) => + enabled ? resolveArtifact({ service }) : Effect.succeed({ image: "", version: "" }); + const [auth, storage, realtime] = yield* Effect.all([ + image("auth", input.setup.authEnabledForSetup), + image("storage", input.setup.storageEnabledForSetup), + image("realtime", input.setup.realtimeEnabledForSetup), + ]); + const keyInputs: ShadowCacheKeyInputs = { + postgresImage: JSON.stringify(["stack-v1", runtime, platform, arch, postgres.image]), + majorVersion: input.setup.majorVersion, + jwtSecret: input.jwtSecret, + jwtExpiry: input.jwtExpiry, + rootKey: input.rootKey ?? "", + dbPassword: input.password, + storageTargetMigration: "", + dbSettings: input.db.settings, + autoExposeNewTables: input.setup.apiAutoExposeNewTables, + webhooksEnabled: resolveSetupWebhooksEnabled(webhooks, input.setup.webhooksEnabled), + rolesSql, + vault: input.setup.vault, + jwks: "", + services: { + auth: { enabled: input.setup.authEnabledForSetup, image: auth.image }, + storage: { enabled: input.setup.storageEnabledForSetup, image: storage.image }, + realtime: { enabled: input.setup.realtimeEnabledForSetup, image: realtime.image }, + }, + }; + const key = shadowCacheKey(keyInputs); + return { + key, + rolesSql, + } satisfies StackShadowCacheEntry; +}); + +export const stackShadowCacheRoles = readRoles; diff --git a/apps/cli/src/command-internal/stack-shadow.integration.test.ts b/apps/cli/src/command-internal/stack-shadow.integration.test.ts index 4c538f9e3d..c67aa94faf 100644 --- a/apps/cli/src/command-internal/stack-shadow.integration.test.ts +++ b/apps/cli/src/command-internal/stack-shadow.integration.test.ts @@ -2,14 +2,14 @@ import { StackError, type Stack } from "@supabase/stack/effect"; import { CliConfigSchema } from "@supabase/config"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Effect, FileSystem, Layer, Option, Path, Schema } from "effect"; +import { Deferred, Effect, FileSystem, Layer, Option, Path, Ref, Schema } from "effect"; import { mockOutput } from "../../tests/helpers/mocks.ts"; import { mockCommandSettings } from "../../tests/helpers/command-mocks.ts"; import { runtimeInfoLayer } from "../shared/runtime/runtime-info.layer.ts"; import { DbConnection } from "./db-connection.service.ts"; import { dbConnectionLayer } from "./db-connection.layer.ts"; import { StackApi, stackApiLayer } from "./stack-api.ts"; -import { stackCatalogSetupLayer } from "./stack-catalog-setup.ts"; +import { StackCatalogSetup, stackCatalogSetupLayer } from "./stack-catalog-setup.ts"; import { parseConnectionString } from "./db-config.parse.ts"; import { stackAcquireShadowDatabase, @@ -20,18 +20,69 @@ import type { ShadowSetupInput } from "./db-bootstrap/shadow-database.ts"; const defaultConfig = Schema.decodeUnknownSync(CliConfigSchema)({}); -const recordingApi = (roots: string[], handles: Stack[]) => +const recordingApi = ( + roots: string[], + handles: Stack[], + restoreFailures = 0, + warmReadyFailures = 0, +) => Layer.effect( StackApi, Effect.gen(function* () { const api = yield* StackApi; + const failures = yield* Ref.make(restoreFailures); + const readyFailures = yield* Ref.make(warmReadyFailures); return StackApi.of({ ...api, create: Effect.fn("ShadowTest.create")((options) => - Effect.sync(() => { - roots.push(options.projectRoot); - }).pipe( + Effect.sync(() => roots.push(options.projectRoot)).pipe( Effect.andThen(api.create(options)), + Effect.map((stack) => ({ + ...stack, + services: { + ...stack.services, + create: Effect.fn("ShadowTest.createService")((creation) => + stack.services.create(creation).pipe( + Effect.flatMap((instance) => { + if (instance.service !== "database") return Effect.succeed(instance); + return Effect.gen(function* () { + const restored = yield* Ref.make(false); + return { + ...instance, + ready: Effect.gen(function* () { + if (yield* Ref.get(restored)) { + const remaining = yield* Ref.getAndUpdate(readyFailures, (value) => + Math.max(0, value - 1), + ); + if (remaining > 0) + return yield* new StackError({ + operation: "ready", + message: "injected ready failure", + }); + } + yield* instance.ready; + }), + restoreSnapshot: (key: string) => + Effect.gen(function* () { + const remaining = yield* Ref.getAndUpdate(failures, (value) => + Math.max(0, value - 1), + ); + if (remaining > 0) + return yield* new StackError({ + operation: "restoreSnapshot", + message: "injected restore failure", + }); + const restoredSnapshot = yield* instance.restoreSnapshot(key); + yield* Ref.set(restored, restoredSnapshot); + return restoredSnapshot; + }), + }; + }); + }), + ), + ), + }, + })), Effect.tap((stack) => Effect.sync(() => { handles.push(stack); @@ -46,6 +97,7 @@ const input = ( fs: FileSystem.FileSystem, path: Path.Path, workdir: string, + setupOverrides: Partial["setup"]> = {}, ): ShadowSetupInput => ({ db: { major_version: 17, settings: {} }, experimental: defaultConfig.experimental, @@ -85,6 +137,7 @@ const input = ( webhooksEnabled: false, apiAutoExposeNewTables: Option.none(), vault: [], + ...setupOverrides, }, }); @@ -98,7 +151,7 @@ const query = Effect.fn("ShadowTest.query")(function* (url: string, sql: string) describe("stack shadow databases", () => { it.live( - "isolates two fresh databases, replays migrations, and removes each owned namespace", + "publishes and restores a baseline while keeping migrations and ownership isolated", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -124,26 +177,132 @@ describe("stack shadow databases", () => { const setup = input(fs, path, root); yield* Effect.scoped( Effect.gen(function* () { + const cold = yield* stackAcquireShadowDatabase(setup, { runtime: "native" }); + expect(cold.restoredFromSnapshot).toBe(false); + expect(cold.snapshotKey).toMatch(/^[0-9a-f]{16}$/u); + expect( + yield* query(cold.url, "SELECT rolname FROM pg_roles WHERE rolname = 'anon'"), + ).toEqual([{ rolname: "anon" }]); + expect( + yield* query(cold.url, "SELECT to_regclass('public.shadow_probe') AS table_name"), + ).toEqual([{ table_name: null }]); + yield* stackMigrateShadow(cold, setup); + expect(yield* query(cold.url, "SELECT value FROM public.shadow_probe")).toEqual([ + { value: "migrated" }, + ]); + }), + ); + yield* Effect.scoped( + Effect.gen(function* () { + const warm = yield* stackAcquireShadowDatabase(setup, { runtime: "native" }); + expect(warm.restoredFromSnapshot).toBe(true); + expect(warm.snapshotKey).toMatch(/^[0-9a-f]{16}$/u); + expect(warm.snapshotKey).toBeDefined(); + expect( + yield* query(warm.url, "SELECT rolname FROM pg_roles WHERE rolname = 'anon'"), + ).toEqual([{ rolname: "anon" }]); + expect( + yield* query(warm.url, "SELECT to_regclass('public.shadow_probe') AS table_name"), + ).toEqual([{ table_name: null }]); + yield* stackMigrateShadow(warm, setup); + expect(yield* query(warm.url, "SELECT value FROM public.shadow_probe")).toEqual([ + { value: "migrated" }, + ]); + }), + ); + + yield* Effect.scoped( + Effect.gen(function* () { + const concurrentSetup = { ...setup, password: "parallel-postgres" }; + const catalog = yield* StackCatalogSetup; + const arrivals = yield* Ref.make(0); + const gate = yield* Deferred.make(); + const barrierCatalog = StackCatalogSetup.of({ + ...catalog, + apply: (catalogInput) => + Effect.gen(function* () { + const count = yield* Ref.updateAndGet(arrivals, (value) => value + 1); + if (count === 2) yield* Deferred.succeed(gate, undefined); + else yield* Deferred.await(gate); + yield* catalog.apply(catalogInput); + }), + }); const [first, second] = yield* Effect.all( [ - stackAcquireShadowDatabase(setup, { runtime: "native" }), - stackAcquireShadowDatabase(setup, { runtime: "native" }), + stackAcquireShadowDatabase(concurrentSetup, { runtime: "native" }).pipe( + Effect.provideService(StackCatalogSetup, barrierCatalog), + ), + stackAcquireShadowDatabase(concurrentSetup, { runtime: "native" }).pipe( + Effect.provideService(StackCatalogSetup, barrierCatalog), + ), ], { concurrency: "unbounded" }, ); expect(first.stack.id).not.toBe(second.stack.id); expect(first.database.id).not.toBe(second.database.id); expect(first.port).not.toBe(second.port); + for (const handle of [first, second]) { + const status = yield* handle.database.status; + const sql = status.endpoints.find((endpoint) => endpoint.name === "sql"); + if (sql === undefined) return yield* Effect.die("missing SQL endpoint"); + const credentials = yield* handle.database.credentials({ from: "host" }); + if (credentials.databaseUrl === undefined) + return yield* Effect.die("missing database credentials"); + const connection = parseConnectionString(credentials.databaseUrl); + if (connection === undefined) return yield* Effect.die("invalid database URL"); + expect(sql.port).toBe(handle.port); + expect(connection.port).toBe(sql.port); + } + expect(first.restoredFromSnapshot).toBe(false); + expect(second.restoredFromSnapshot).toBe(false); + const publishedKeys = [first.snapshotKey, second.snapshotKey].filter( + (key): key is string => key !== undefined, + ); + expect(publishedKeys).toHaveLength(2); + expect(publishedKeys[0]).toMatch(/^[0-9a-f]{16}$/u); + expect(publishedKeys[1]).toBe(publishedKeys[0]); + }), + ); - yield* stackMigrateShadow(first, setup); - expect(yield* query(first.url, "SELECT value FROM public.shadow_probe")).toEqual([ - { value: "migrated" }, - ]); + const recoveredKey = yield* Effect.scoped( + Effect.gen(function* () { + const recovered = yield* stackAcquireShadowDatabase(setup, { runtime: "native" }); + expect(recovered.restoredFromSnapshot).toBe(true); + expect(recovered.snapshotKey).toBeDefined(); + expect( + yield* query(recovered.url, "SELECT rolname FROM pg_roles WHERE rolname = 'anon'"), + ).toEqual([{ rolname: "anon" }]); expect( - yield* query(second.url, "SELECT to_regclass('public.shadow_probe') AS table_name"), + yield* query( + recovered.url, + "SELECT to_regclass('public.shadow_probe') AS table_name", + ), ).toEqual([{ table_name: null }]); + return recovered.snapshotKey; }), ); + if (recoveredKey === undefined) return yield* Effect.die("missing recovery key"); + + yield* fs.writeFileString( + path.join(root, "supabase", "roles.sql"), + "CREATE ROLE cache_probe;\n", + ); + const rolesKey = yield* Effect.scoped( + Effect.gen(function* () { + const rolesChanged = yield* stackAcquireShadowDatabase(setup, { runtime: "native" }); + expect(rolesChanged.restoredFromSnapshot).toBe(false); + expect(rolesChanged.snapshotKey).toBeDefined(); + expect(rolesChanged.snapshotKey).not.toBe(recoveredKey); + expect( + yield* query( + rolesChanged.url, + "SELECT rolname FROM pg_roles WHERE rolname = 'cache_probe'", + ), + ).toEqual([{ rolname: "cache_probe" }]); + return rolesChanged.snapshotKey; + }), + ); + if (rolesKey === undefined) return yield* Effect.die("missing roles key"); for (const shadowRoot of roots) expect(yield* fs.exists(shadowRoot)).toBe(false); for (const handle of handles) expect(yield* Effect.flip(handle.services.list)).toBeInstanceOf(StackError); @@ -152,7 +311,110 @@ describe("stack shadow databases", () => { expect(yield* api.discover({ stateRoot: path.join(root, "stacks") })).toEqual([]); }).pipe(Effect.provide(layers)); }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), - 120_000, + 420_000, + ); + + it.live( + "bypasses and disables the cache without publication or restoration", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-shadow-cache-gate-" }); + const output = mockOutput(); + const roots: string[] = []; + const handles: Stack[] = []; + const layers = Layer.mergeAll( + recordingApi(roots, handles), + stackCatalogSetupLayer, + dbConnectionLayer, + runtimeInfoLayer, + mockCommandSettings({ workdir: root, supabaseHome: root }), + output.layer, + ); + yield* Effect.gen(function* () { + const setup = input(fs, path, root); + yield* Effect.scoped(stackAcquireShadowDatabase(setup, { runtime: "native" })); + + const bypassed = yield* Effect.scoped( + stackAcquireShadowDatabase(setup, { runtime: "native", bypassCache: true }), + ); + expect(bypassed.restoredFromSnapshot).toBe(false); + expect(bypassed.snapshotKey).toBeUndefined(); + + const optOut = input(fs, path, root, { + projectEnvValues: { SUPABASE_SHADOW_CACHE: "0" }, + }); + const disabled = yield* Effect.scoped( + stackAcquireShadowDatabase(optOut, { runtime: "native" }), + ); + expect(disabled.restoredFromSnapshot).toBe(false); + expect(disabled.snapshotKey).toBeUndefined(); + }).pipe(Effect.provide(layers)); + for (const shadowRoot of roots) expect(yield* fs.exists(shadowRoot)).toBe(false); + expect(output.stderrText).not.toContain("Failed to destroy"); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + 180_000, + ); + + it.live( + "recreates and republishes after a restored database fails readiness", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-shadow-restore-fallback-", + }); + const output = mockOutput(); + const setup = input(fs, path, root); + const roots: string[] = []; + const handles: Stack[] = []; + const layers = Layer.mergeAll( + recordingApi(roots, handles, 0, 1), + stackCatalogSetupLayer, + dbConnectionLayer, + runtimeInfoLayer, + mockCommandSettings({ workdir: root, supabaseHome: root }), + output.layer, + ); + yield* Effect.gen(function* () { + const first = yield* Effect.scoped( + stackAcquireShadowDatabase(setup, { runtime: "native" }), + ); + expect(first.restoredFromSnapshot).toBe(false); + const recoveredKey = yield* Effect.scoped( + Effect.gen(function* () { + const recovered = yield* stackAcquireShadowDatabase(setup, { runtime: "native" }); + expect(recovered.restoredFromSnapshot).toBe(false); + expect(recovered.snapshotKey).toBeDefined(); + const status = yield* recovered.database.status; + const sql = status.endpoints.find((endpoint) => endpoint.name === "sql"); + if (sql === undefined) return yield* Effect.die("missing SQL endpoint"); + const credentials = yield* recovered.database.credentials({ from: "host" }); + if (credentials.databaseUrl === undefined) + return yield* Effect.die("missing database credentials"); + const connection = parseConnectionString(credentials.databaseUrl); + if (connection === undefined) return yield* Effect.die("invalid database URL"); + expect(sql.port).toBe(recovered.port); + expect(connection.port).toBe(sql.port); + expect(yield* query(recovered.url, "SELECT 1 AS ready")).toEqual([{ ready: 1 }]); + return recovered.snapshotKey; + }), + ); + expect(output.stderrText).toContain("cached stack shadow baseline unusable"); + + yield* Effect.scoped( + Effect.gen(function* () { + const warm = yield* stackAcquireShadowDatabase(setup, { runtime: "native" }); + expect(warm.restoredFromSnapshot).toBe(true); + expect(warm.snapshotKey).toBe(recoveredKey); + expect(yield* query(warm.url, "SELECT 1 AS ready")).toEqual([{ ready: 1 }]); + }), + ); + }).pipe(Effect.provide(layers)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + 180_000, ); it.live( diff --git a/apps/cli/src/command-internal/stack-shadow.ts b/apps/cli/src/command-internal/stack-shadow.ts index a26d324cde..52e6dd6eff 100644 --- a/apps/cli/src/command-internal/stack-shadow.ts +++ b/apps/cli/src/command-internal/stack-shadow.ts @@ -1,4 +1,4 @@ -import { Effect, FileSystem, Path, Redacted, Schema } from "effect"; +import { Effect, FileSystem, Path, Redacted, Result, Schema } from "effect"; import type { DatabaseInstance, Stack } from "@supabase/stack/effect"; import { CommandSettings } from "../config/command-settings.service.ts"; import { Output } from "../shared/output/output.service.ts"; @@ -17,6 +17,7 @@ import { import type { SetupDatabaseOptions } from "./db-bootstrap/db-setup.ts"; import { listLocalMigrationPaths } from "./migration-history.ts"; import { applyMigrations } from "./migration-apply.ts"; +import { stackShadowCacheEntry, stackShadowCacheRoles } from "./stack-shadow-cache.ts"; type Runtime = "native" | "docker" | "podman"; @@ -28,12 +29,16 @@ export interface StackShadowAcquiredHandle { readonly port: number; readonly runtime: Runtime; readonly version: string; + /** Cache lineage when a baseline was restored or atomically published by this handle. */ + readonly snapshotKey?: string; + readonly restoredFromSnapshot: boolean; } interface ShadowOptions { readonly port?: number; readonly runtime?: Runtime; readonly webhooks?: SetupDatabaseOptions["webhooks"]; + readonly bypassCache?: boolean; } const shadowError = (cause: { readonly message: string }) => @@ -41,6 +46,9 @@ const shadowError = (cause: { readonly message: string }) => ? cause : new ShadowDbError({ message: cause.message, reason: "database" }); +const causeMessage = (cause: unknown): string => + cause instanceof Error ? cause.message : String(cause); + const acquireNamespace = Effect.fn("StackShadow.acquireNamespace")(function* (opts: ShadowOptions) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -64,49 +72,121 @@ const initialize = Effect.fn("StackShadow.initialize")(function* ( input: ShadowSetupInput, opts: ShadowOptions, ) { - const settings = yield* Schema.decodeEffect( + const runtimeInfo = yield* RuntimeInfo; + const dbSettings = yield* Schema.decodeEffect( Schema.Record(Schema.String, Schema.Union([Schema.String, Schema.Finite, Schema.Boolean])), )( Object.fromEntries( Object.entries(input.db.settings ?? {}).filter(([, value]) => value !== undefined), ), ).pipe(Effect.mapError(shadowError)); - const database = yield* stack.services.create({ - service: "database", - config: { - version: String(input.setup.majorVersion), - databasePassword: Redacted.make(input.password), - jwtSecret: Redacted.make(input.jwtSecret), - jwtExpiry: input.jwtExpiry, - healthTimeoutMs: input.healthTimeoutSeconds * 1_000, - ...(input.rootKey === undefined ? {} : { rootKey: Redacted.make(input.rootKey) }), - settings, - }, - endpoints: { sql: { port: opts.port ?? "auto" } }, - }); - yield* database.start; - yield* database.ready; + const createDatabase = () => + stack.services.create({ + service: "database", + config: { + version: String(input.setup.majorVersion), + databasePassword: Redacted.make(input.password), + jwtSecret: Redacted.make(input.jwtSecret), + jwtExpiry: input.jwtExpiry, + healthTimeoutMs: input.healthTimeoutSeconds * 1_000, + ...(input.rootKey === undefined ? {} : { rootKey: Redacted.make(input.rootKey) }), + settings: dbSettings, + }, + endpoints: { sql: { port: opts.port ?? "auto" } }, + }); + let database = yield* createDatabase(); + const cacheResolution = yield* Effect.result( + stackShadowCacheEntry( + input, + runtime, + runtimeInfo.platform, + runtimeInfo.arch, + opts.webhooks, + opts.bypassCache, + ), + ); + const output = yield* Output; + const cache = Result.isSuccess(cacheResolution) ? cacheResolution.success : undefined; + if (Result.isFailure(cacheResolution)) + yield* output.raw( + `Warning: stack shadow cache unavailable: ${causeMessage(cacheResolution.failure)}; continuing uncached.\n`, + "stderr", + ); + let restored = false; + if (cache !== undefined) { + const warmAttempt = yield* Effect.result( + database + .restoreSnapshot(cache.key) + .pipe( + Effect.flatMap((restoredSnapshot) => + restoredSnapshot + ? database.start.pipe(Effect.andThen(database.ready), Effect.as(true)) + : Effect.succeed(false), + ), + ), + ); + if (Result.isSuccess(warmAttempt)) { + restored = warmAttempt.success; + if (!restored) { + yield* database.start; + yield* database.ready; + } + } else { + yield* output.raw("Warning: cached stack shadow baseline unusable; recreating.\n", "stderr"); + yield* database.destroy; + database = yield* createDatabase(); + yield* database.start; + yield* database.ready; + } + } else { + yield* database.start; + yield* database.ready; + } const catalog = yield* StackCatalogSetup; - yield* catalog.apply({ - target: { - stack, - database, - databaseServices: [ - ...(input.setup.authEnabledForSetup ? ["auth" as const] : []), - ...(input.setup.storageEnabledForSetup ? ["storage" as const] : []), - ...(input.setup.realtimeEnabledForSetup ? ["realtime" as const] : []), - ], - jwtSecret: input.jwtSecret, - }, - overlay: { - ...(opts.webhooks === undefined ? {} : { webhooks: opts.webhooks }), - webhooksEnabled: input.setup.webhooksEnabled, - apiAutoExposeNewTables: input.setup.apiAutoExposeNewTables, - vault: input.setup.vault, - workdir: input.workdir, - announceRoles: false, - }, - }); + if (!restored) + yield* catalog.apply({ + target: { + stack, + database, + databaseServices: [ + ...(input.setup.authEnabledForSetup ? ["auth" as const] : []), + ...(input.setup.storageEnabledForSetup ? ["storage" as const] : []), + ...(input.setup.realtimeEnabledForSetup ? ["realtime" as const] : []), + ], + jwtSecret: input.jwtSecret, + }, + overlay: { + ...(opts.webhooks === undefined ? {} : { webhooks: opts.webhooks }), + webhooksEnabled: input.setup.webhooksEnabled, + apiAutoExposeNewTables: input.setup.apiAutoExposeNewTables, + vault: input.setup.vault, + workdir: input.workdir, + announceRoles: false, + }, + }); + let snapshotKey = restored ? cache?.key : undefined; + if (!restored && cache !== undefined) { + const rolesAfter = yield* Effect.result(stackShadowCacheRoles(input)); + if (Result.isFailure(rolesAfter)) { + yield* output.raw( + `Warning: stack shadow baseline not cached: unable to read supabase/roles.sql (${causeMessage(rolesAfter.failure)}).\n`, + "stderr", + ); + } else if (rolesAfter.success === cache.rolesSql) { + const published = yield* Effect.result( + database.stop.pipe(Effect.flatMap(() => database.saveSnapshot(cache.key))), + ); + yield* database.start.pipe(Effect.andThen(database.ready)); + if (Result.isSuccess(published)) { + snapshotKey = cache.key; + } else { + yield* output.raw( + `Warning: stack shadow baseline not cached: ${causeMessage(published.failure)}.\n`, + "stderr", + ); + } + } + } const credentials = yield* database.credentials({ from: "host" }); const credentialUrl = credentials.databaseUrl; const credentialsConn = @@ -126,6 +206,8 @@ const initialize = Effect.fn("StackShadow.initialize")(function* ( port: conn.port, runtime, version: String(input.setup.majorVersion), + snapshotKey, + restoredFromSnapshot: restored, } satisfies StackShadowAcquiredHandle; }); diff --git a/apps/cli/src/commands/db/diff/SIDE_EFFECTS.md b/apps/cli/src/commands/db/diff/SIDE_EFFECTS.md index df124dc67b..1529a7a36f 100644 --- a/apps/cli/src/commands/db/diff/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/diff/SIDE_EFFECTS.md @@ -10,8 +10,8 @@ pending port. When `[experimental].stack` is on, each shadow is a fresh database in an invocation-owned, unique temporary stack namespace. The command applies the catalog and project migrations as needed, -then destroys its namespace when the Effect scope closes. Shadow baselines are not cached; -the snapshot cache described below belongs to the legacy container backend. Native artifacts +then destroys its namespace when the Effect scope closes. Stack shadows use the stack baseline +cache described below. Native artifacts are shared through `$SUPABASE_HOME/cache/stack`; shadow state and data use the normal stack registry, so `stack list` and `stack destroy` can find a shadow left by an abrupt CLI exit. Each shadow owns a unique temporary project root and uses an automatically assigned port; `db.shadow_port` applies only to the legacy backend. Migra, pgAdmin, and `--use-pg-schema` are rejected in stack mode. @@ -34,6 +34,7 @@ it, and JSON `null` disables formatting without disabling safe compaction. | `/supabase/migrations/*.sql` | SQL | shadow provisioning (applied to the shadow source) — `--use-pgadmin` too, via the SAME `migrateShadowDatabase` | | `/supabase/roles.sql` | SQL | shadow provisioning, PG14 and PG15 alike (unlike `db reset`'s PG15-only local path); also hashed into the shadow-baseline cache key on every cache-eligible acquire, warm hits included (where no baseline is applied at all); missing file tolerated | | `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar` | tar | warm shadow-cache hit — the matching snapshot is streamed into the fresh shadow; every cache-eligible acquire (warm hit and successful cold export) also enumerates and `stat`s every `shadow-baseline-*.tar` for LRU keep-3 + 2-day mtime TTL and may delete other keys (`SUPABASE_HOME` overrides the `~/.supabase` root) | +| managed snapshot key `` | backend | warm stack-shadow cache hit; managed retention may evict entries and snapshots survive stack destruction | | `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar..partial` | tar | abandoned-partial sweep on every cache-eligible acquire (warm hit and cold export) — enumerated and `stat`ed, and removed when older than 5 minutes (a crashed/SIGKILLed earlier export's leftover) | | `[db.migrations].schema_paths` globs / `/supabase/database/**` / `/supabase/schemas/**` | SQL | migra engine only, for the local-target declarative-schema fallback; pg-delta always compares the migrations baseline directly to the live target | | `~/.supabase/access-token` | plain text | `--linked` / `--db-url` with no `SUPABASE_ACCESS_TOKEN` | @@ -41,15 +42,16 @@ it, and JSON `null` disables formatting without disabling safe compaction. ## Files Written -| Path | Format | When | -| --------------------------------------------------------------------------- | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `/supabase/migrations/_.sql` | SQL | non-empty `--file` diff; bundled pg-delta may emit ordered transaction-aware files, while pgAdmin always emits one | -| `` (from `--output` / `-o`) | SQL | explicit `--from/--to` mode with `--output`; flattened review representation, not a portable apply script | -| `/supabase/.temp/pgdelta/v2/debug//*.json` | JSON | bundled engine with `PGDELTA_DEBUG` | -| `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar` | tar | cache-enabled COLD shadow provision creates the current key's snapshot (native diff targets + the explicit `--from/--to migrations` shadow; never `--use-pgadmin`/`--use-pg-schema`); a warm hit `touch`es its mtime (LRU); every cache-eligible acquire may delete other keys under LRU keep-3 + 2-day mtime TTL — ~90MB (`SUPABASE_HOME` overrides the root) | -| `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar..partial` | tar | during a cold export — the in-flight temp file, `rename`d into the tar above on success and removed on failure; only a crash/SIGKILL leaves it behind, and later cold exports / warm hits sweep leftovers older than 5 minutes | -| `~/.supabase//linked-project.json` | JSON | `--linked` (post-run cache) | -| `~/.supabase/telemetry.json` | JSON | every invocation (post-run) | +| Path | Format | When | +| --------------------------------------------------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `/supabase/migrations/_.sql` | SQL | non-empty `--file` diff; bundled pg-delta may emit ordered transaction-aware files, while pgAdmin always emits one | +| `` (from `--output` / `-o`) | SQL | explicit `--from/--to` mode with `--output`; flattened review representation, not a portable apply script | +| `/supabase/.temp/pgdelta/v2/debug//*.json` | JSON | bundled engine with `PGDELTA_DEBUG` | +| `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar` | tar | cache-enabled COLD shadow provision creates the current key's snapshot (native diff targets + the explicit `--from/--to migrations` shadow; never `--use-pgadmin`/`--use-pg-schema`); a warm hit `touch`es its mtime (LRU); every cache-eligible acquire may delete other keys under LRU keep-3 + 2-day mtime TTL — ~90MB (`SUPABASE_HOME` overrides the root) | +| managed snapshot key `` | backend | stack-mode cache-enabled cold shadow provision saves the database baseline; the last completed save replaces the key | +| `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar..partial` | tar | during a cold export — the in-flight temp file, `rename`d into the tar above on success and removed on failure; only a crash/SIGKILL leaves it behind, and later cold exports / warm hits sweep leftovers older than 5 minutes | +| `~/.supabase//linked-project.json` | JSON | `--linked` (post-run cache) | +| `~/.supabase/telemetry.json` | JSON | every invocation (post-run) | ## Docker @@ -100,7 +102,7 @@ of this command's own target resolve, ahead of the differ container. | `SUPABASE_PROJECT_ID` | overrides the shadow container's project id/labels, same as `db start`/`db reset` (`utils.DbId`); ALSO the linked-ref resolution fallback `--project-ref` supersedes — see Notes for the narrower scope of the flag | no | | `SUPABASE_NETWORK_ID` (`--network-id`) | forces the shadow container/network onto an existing Docker network | no | | `SUPABASE_HOME` | overrides the `~/.supabase` root used for the shadow baseline cache (and other CLI state) | no | -| `SUPABASE_SHADOW_CACHE` | shadow baseline cache; on by default, opt-out (`0`/`false`); the shadow's post-baseline PGDATA is snapshotted to a tar and restored into the next run's fresh container (see Notes) | no | +| `SUPABASE_SHADOW_CACHE` | shadow baseline cache; on by default, opt-out (`0`/`false`); the shadow's post-baseline state is saved under a managed snapshot key and restored into the next run's fresh stack database (see Notes) | no | | `SUPABASE_EXPERIMENTAL_PG_DELTA` | force pg-delta engine | no | | `PGDELTA_DEBUG` | pg-delta debug capture | no | | `SUPABASE_SSL_DEBUG` | migra SSL debug logging | no | @@ -224,6 +226,13 @@ transaction metadata. ### Shadow baseline cache (`SUPABASE_SHADOW_CACHE`, default ON) +In stack mode, cache-eligible shadows use a managed snapshot keyed by ``; +the key includes resolved stack artifacts, runtime/platform, enabled catalog services, settings, +credentials, overlay inputs, and `roles.sql`. Warm restores touch the entry, cold provisioning +publishes only when the roles file is unchanged, and retention keeps three entries by LRU. +A falsy `SUPABASE_SHADOW_CACHE` bypasses this cache. A failed warm restore recreates the database; +a failed cache export warns and continues with the live shadow. + On by default; setting `SUPABASE_SHADOW_CACHE` to anything not viper-true (`false`/`0`/empty/garbage, honored from the ambient env AND the project's dotenv, e.g. `supabase/.env`) turns it off, restoring the documented uncached lifecycle. A warm hit diff --git a/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts b/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts new file mode 100644 index 0000000000..79869c0956 --- /dev/null +++ b/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts @@ -0,0 +1,142 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Path, Redacted } from "effect"; +import { FetchHttpClient } from "effect/unstable/http"; +import { create as createStack } from "@supabase/stack/effect"; +import { tmpdir } from "node:os"; +import { runSupabaseEffect } from "../../../../tests/helpers/cli.ts"; + +const COMMAND_TIMEOUT_MS = 8 * 60_000; +const TEST_TIMEOUT_MS = COMMAND_TIMEOUT_MS * 4 + 2 * 60_000; +const nativeSupported = + (process.platform === "linux" && (process.arch === "x64" || process.arch === "arm64")) || + (process.platform === "darwin" && process.arch === "arm64"); + +const config = `project_id = "db-diff-stack-cache-e2e" + +[experimental] +stack = true + +[db] +major_version = 17 + +[auth] +enabled = false + +[storage] +enabled = false + +[realtime] +enabled = false +`; + +const composeStack = Effect.fn("DbDiffStackCacheE2e.composeStack")(function* ( + root: string, + home: string, + runtime: "native" | "docker", +) { + const stack = yield* createStack({ + projectRoot: root, + stateRoot: `${home}/stacks`, + cacheRoot: `${home}/cache/stack`, + runtime, + }); + yield* Effect.addFinalizer(() => + stack.destroy.pipe( + Effect.catch((cause) => Effect.die(new Error(`stack cleanup failed: ${cause.message}`))), + ), + ); + yield* stack.composition.supabase([ + { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make("db-diff-stack-cache-e2e-jwt-secret-with-32-chars"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }, + ]); + yield* stack.composition.start; + return stack; +}); + +const command = (args: ReadonlyArray, cwd: string, home: string) => + runSupabaseEffect([...args], { + cwd, + home, + env: { SUPABASE_EXPERIMENTAL_STACK: "1", SUPABASE_SHADOW_CACHE: undefined }, + exitTimeoutMs: COMMAND_TIMEOUT_MS, + }); + +const assertSuccess = ( + result: { exitCode: number; stdout: string; stderr: string }, + name: string, +) => { + expect(result.exitCode, `${name}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`).toBe(0); +}; + +describe("supabase db diff (stack shadow baseline cache)", () => { + for (const runtime of ["native", "docker"] as const) { + if (runtime === "native" && !nativeSupported) continue; + + it.live( + `reuses a ${runtime} baseline while replaying changed project migrations`, + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const project = yield* fs.makeTempDirectoryScoped({ + prefix: `db-diff-cache-${runtime}-`, + }); + const home = yield* fs.makeTempDirectoryScoped({ + prefix: `db-diff-cache-home-${runtime}-`, + }); + const sharedArtifacts = path.join(tmpdir(), "supabase-stack-artifacts"); + yield* fs.makeDirectory(sharedArtifacts, { recursive: true }); + yield* fs.makeDirectory(path.join(home, "cache"), { recursive: true }); + yield* fs.symlink(sharedArtifacts, path.join(home, "cache", "stack")); + + const supabase = path.join(project, "supabase"); + const migrations = path.join(supabase, "migrations"); + yield* fs.makeDirectory(migrations, { recursive: true }); + yield* fs.writeFileString(path.join(supabase, "config.toml"), config); + const migrationPath = path.join(migrations, "20260921000000_cache_story.sql"); + yield* fs.writeFileString( + migrationPath, + "create table public.stack_cache_first (value text primary key);\n", + ); + + const stack = yield* composeStack(project, home, runtime); + + const first = yield* command( + ["db", "diff", "--from", "migrations", "--to", "local", "--use-pg-delta"], + project, + home, + ); + assertSuccess(first, `${runtime} cold db diff`); + expect(first.stdout).toMatch(/stack_cache_first/iu); + + yield* fs.writeFileString( + migrationPath, + "create table public.stack_cache_second (value text primary key);\n", + ); + const second = yield* command( + ["db", "diff", "--from", "migrations", "--to", "local", "--use-pg-delta"], + project, + home, + ); + assertSuccess(second, `${runtime} warm db diff`); + expect(second.stdout).toMatch(/stack_cache_second/iu); + expect(second.stdout).not.toMatch(/stack_cache_first/iu); + + const stackEntries = (yield* fs.readDirectory(path.join(home, "stacks"))).filter( + (entry) => /^[0-9a-f]{64}$/u.test(entry), + ); + expect(stackEntries).toEqual([stack.id]); + }).pipe(Effect.scoped, Effect.provide([BunServices.layer, FetchHttpClient.layer])), + { timeout: TEST_TIMEOUT_MS }, + ); + } +}); diff --git a/apps/cli/src/commands/db/dump/dump.integration.test.ts b/apps/cli/src/commands/db/dump/dump.integration.test.ts index 5c95a3b6aa..553c17868d 100644 --- a/apps/cli/src/commands/db/dump/dump.integration.test.ts +++ b/apps/cli/src/commands/db/dump/dump.integration.test.ts @@ -95,7 +95,7 @@ const managedDumpStackApi = (runtime: "native" | "docker") => { logs: Stream.empty, credentials: () => Effect.succeed({ databaseUrl: "postgresql://postgres:secret@127.0.0.1:54322/postgres" }), - exportSnapshot: () => Effect.die("unused"), + saveSnapshot: () => Effect.die("unused"), restoreSnapshot: () => Effect.die("unused"), resetData: Effect.die("unused"), }; diff --git a/apps/cli/src/commands/db/pull/SIDE_EFFECTS.md b/apps/cli/src/commands/db/pull/SIDE_EFFECTS.md index de70163154..b159d490d2 100644 --- a/apps/cli/src/commands/db/pull/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/pull/SIDE_EFFECTS.md @@ -16,8 +16,8 @@ runs the same in-process declarative export (`supabase/schemas` plus When `[experimental].stack` is on, each shadow is a fresh database in an invocation-owned, unique temporary stack namespace. The command applies the catalog and project migrations as needed, -then destroys its namespace when the Effect scope closes. Shadow baselines are not cached; -the snapshot cache described below belongs to the legacy container backend. Native artifacts +then destroys its namespace when the Effect scope closes. Stack shadows use the stack baseline +cache described below. Native artifacts are shared through `$SUPABASE_HOME/cache/stack`; shadow state and data use the normal stack registry, so `stack list` and `stack destroy` can find a shadow left by an abrupt CLI exit. Each shadow owns a unique temporary project root and uses an automatically assigned port; `db.shadow_port` applies only to the legacy backend. Migra (`--diff-engine migra`) is rejected in stack mode. @@ -39,23 +39,25 @@ disables formatting without disabling safe compaction. | `/supabase/migrations/*.sql` | SQL | history reconciliation + shadow provisioning | | `/supabase/roles.sql` | SQL | migration-style pull only (`--declarative` provisions no shadow); also hashed into the shadow-baseline cache key on every cache-eligible acquire, warm hits included (where no baseline is applied at all); missing file tolerated | | `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar` | tar | warm shadow-cache hit (migration-style pull) — the matching snapshot is streamed into the fresh shadow; every cache-eligible acquire (warm hit and successful cold export) also enumerates and `stat`s every `shadow-baseline-*.tar` for LRU keep-3 + 2-day mtime TTL and may delete other keys (`SUPABASE_HOME` overrides the `~/.supabase` root) | +| managed snapshot key `` | backend | warm stack-shadow cache hit; managed retention may evict entries and snapshots survive stack destruction | | `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar..partial` | tar | abandoned-partial sweep on every cache-eligible acquire (warm hit and cold export) — enumerated and `stat`ed, and removed when older than 5 minutes (a crashed/SIGKILLed earlier export's leftover) | | `~/.supabase/access-token` | plain text | linked target with no `SUPABASE_ACCESS_TOKEN` | | `/supabase/.temp/project-ref` | plain text | linked ref resolution — skipped when `--project-ref` (or `SUPABASE_PROJECT_ID`) is set | ## Files Written -| Path | Format | When | -| --------------------------------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `/supabase/migrations/_.sql` | SQL | migration-style pull (non-empty diff, or the initial-migra `pg_dump` seed) | -| `/supabase/schemas/**` | SQL | `--declarative` or deprecated `--experimental` export | -| `/supabase/schemas/.pgdelta-export.json` | JSON | bundled declarative / deprecated-`--experimental` export metadata | -| `/supabase/config.toml` | TOML | declarative export updates `[db.migrations].schema_paths` when `[experimental.pgdelta] enabled` resolves false (section absent, `enabled` omitted, or `enabled = false` — the default); skipped when `enabled = true` | -| `/supabase/.temp/pgdelta/v2/debug//*.json` | JSON | bundled engine with `PGDELTA_DEBUG` | -| `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar` | tar | cache-enabled COLD shadow provision creates the current key's snapshot, migration-style pull only (never a declarative / deprecated-`--experimental` export, which provisions no shadow); a warm hit `touch`es its mtime (LRU); every cache-eligible acquire may delete other keys under LRU keep-3 + 2-day mtime TTL — ~90MB (`SUPABASE_HOME` overrides the root) | -| `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar..partial` | tar | during a cold export — the in-flight temp file, `rename`d into the tar above on success and removed on failure; only a crash/SIGKILL leaves it behind, and later cold exports / warm hits sweep leftovers older than 5 minutes | -| `~/.supabase//linked-project.json` | JSON | linked (post-run cache) | -| `~/.supabase/telemetry.json` | JSON | every invocation (post-run) | +| Path | Format | When | +| --------------------------------------------------------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `/supabase/migrations/_.sql` | SQL | migration-style pull (non-empty diff, or the initial-migra `pg_dump` seed) | +| `/supabase/schemas/**` | SQL | `--declarative` or deprecated `--experimental` export | +| `/supabase/schemas/.pgdelta-export.json` | JSON | bundled declarative / deprecated-`--experimental` export metadata | +| `/supabase/config.toml` | TOML | declarative export updates `[db.migrations].schema_paths` when `[experimental.pgdelta] enabled` resolves false (section absent, `enabled` omitted, or `enabled = false` — the default); skipped when `enabled = true` | +| `/supabase/.temp/pgdelta/v2/debug//*.json` | JSON | bundled engine with `PGDELTA_DEBUG` | +| `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar` | tar | cache-enabled COLD shadow provision creates the current key's snapshot, migration-style pull only (never a declarative / deprecated-`--experimental` export, which provisions no shadow); a warm hit `touch`es its mtime (LRU); every cache-eligible acquire may delete other keys under LRU keep-3 + 2-day mtime TTL — ~90MB (`SUPABASE_HOME` overrides the root) | +| managed snapshot key `` | backend | stack-mode cache-enabled cold shadow provision saves the database baseline; the last completed save replaces the key | +| `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar..partial` | tar | during a cold export — the in-flight temp file, `rename`d into the tar above on success and removed on failure; only a crash/SIGKILL leaves it behind, and later cold exports / warm hits sweep leftovers older than 5 minutes | +| `~/.supabase//linked-project.json` | JSON | linked (post-run cache) | +| `~/.supabase/telemetry.json` | JSON | every invocation (post-run) | ## Docker @@ -72,6 +74,13 @@ disables formatting without disabling safe compaction. ### Shadow baseline cache (`SUPABASE_SHADOW_CACHE`, default ON) +In stack mode, cache-eligible shadows use a managed snapshot keyed by ``; +the key includes resolved stack artifacts, runtime/platform, enabled catalog services, settings, +credentials, overlay inputs, and `roles.sql`. Warm restores touch the entry, cold provisioning +publishes only when the roles file is unchanged, and retention keeps three entries by LRU. +A falsy `SUPABASE_SHADOW_CACHE` bypasses this cache. A failed warm restore recreates the database; +a failed cache export warns and continues with the live shadow. + On by default; setting `SUPABASE_SHADOW_CACHE` to anything not viper-true (`false`/`0`/empty/garbage, honored from the ambient env AND the project's dotenv, e.g. `supabase/.env`) turns it off, restoring the documented uncached lifecycle. A warm hit @@ -116,7 +125,7 @@ at all, so nothing is cached for it. | `SUPABASE_NETWORK_ID` (`--network-id`) | forces the shadow container/network onto an existing Docker network | no | | `SUPABASE_USE_SLIM_IMAGES` | resolves the current-pin shadow Postgres, `pg_dump`, PG15+ realtime/storage/auth migrate-job images (migration-style cold shadow), and (for migra) the edge-runtime image from the slim `ghcr.io/supabase/cli` builds (`true`/`1` enable); majors 13/15 use `15.14.1.167` when the flag is on; historical pins, PG14, OrioleDB, flag-off `15.8.1.085`, and `deno_version = 1` stay on docker.io | no | | `SUPABASE_HOME` | overrides the `~/.supabase` root used for the shadow baseline cache (and other CLI state) | no | -| `SUPABASE_SHADOW_CACHE` | shadow baseline cache; on by default, opt-out (`0`/`false`); the shadow's post-baseline PGDATA is snapshotted to a tar and restored into the next run's fresh container (see Notes) | no | +| `SUPABASE_SHADOW_CACHE` | shadow baseline cache; on by default, opt-out (`0`/`false`); the shadow's post-baseline state is saved under a managed snapshot key and restored into the next run's fresh stack database (see Notes) | no | | `SUPABASE_EXPERIMENTAL_PG_DELTA` | force pg-delta diff engine | no | | `SUPABASE_EXPERIMENTAL` | selects the deprecated in-process structured-dump export (same as `--declarative`) when `--declarative` is not set | no | diff --git a/apps/cli/src/commands/db/reset/reset.integration.test.ts b/apps/cli/src/commands/db/reset/reset.integration.test.ts index 87e16dd2f2..33132f92d1 100644 --- a/apps/cli/src/commands/db/reset/reset.integration.test.ts +++ b/apps/cli/src/commands/db/reset/reset.integration.test.ts @@ -549,7 +549,7 @@ const stackService = ( service: "database", credentials: () => Effect.succeed({ databaseUrl: "postgresql://postgres:postgres@127.0.0.1:5432/postgres" }), - exportSnapshot: () => Effect.die("unused"), + saveSnapshot: () => Effect.die("unused"), restoreSnapshot: () => Effect.die("unused"), resetData: overrides.resetData ?? Effect.die("unused"), }; diff --git a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts index c50002d8af..ecca07a352 100644 --- a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts @@ -100,7 +100,7 @@ function generateStackApi(workdir: string) { destroy: unusedStack, prepare: unusedStack, credentials: unusedStackFn, - exportSnapshot: unusedStackFn, + saveSnapshot: unusedStackFn, restoreSnapshot: unusedStackFn, resetData: unusedStack, logs: Stream.empty, diff --git a/apps/cli/src/commands/db/schema/declarative/sync/SIDE_EFFECTS.md b/apps/cli/src/commands/db/schema/declarative/sync/SIDE_EFFECTS.md index 155c9b64ed..0e04a63815 100644 --- a/apps/cli/src/commands/db/schema/declarative/sync/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/schema/declarative/sync/SIDE_EFFECTS.md @@ -7,8 +7,8 @@ When `[experimental].stack` is on, each shadow uses a fresh, invocation-owned st with an automatically assigned port. State and data live under `$SUPABASE_HOME/stacks`, and native artifacts are shared through `$SUPABASE_HOME/cache/stack`. The command destroys its shadow namespaces when its Effect scope closes. An abrupt process exit can leave a namespace -visible to `stack list` for manual `stack destroy` cleanup. Stack shadows do not use the -baseline snapshot cache described below; `--no-cache` affects only the legacy backend. +visible to `stack list` for manual `stack destroy` cleanup. Stack shadows use the stack baseline +cache described below; `--no-cache` bypasses it as well as the legacy backend cache. Pg-delta runs in-process and uses two scoped shadow databases. Coverage gaps warn; `--strict-coverage` makes @@ -54,7 +54,7 @@ disabling safe compaction. | Variable | Purpose | Required? | | ---------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | | `SUPABASE_HOME` | overrides the `~/.supabase` root used for the shadow baseline cache (and other CLI state) | no | -| `SUPABASE_SHADOW_CACHE` | shadow baseline cache; on by default, opt-out (`0`/`false`); the shadow's post-baseline PGDATA is snapshotted to a tar and restored into the next run's fresh container (see Notes) | no | +| `SUPABASE_SHADOW_CACHE` | shadow baseline cache; on by default, opt-out (`0`/`false`); the shadow's post-baseline state is saved under a managed snapshot key and restored into the next run's fresh stack database (see Notes) | no | | `PGDELTA_DEBUG` | bundled-engine debug artifacts | no | | `SUPABASE_SERVICES_HOSTNAME` | local DB host for the bootstrap generate | no | | `DOCKER_HOST` | tcp daemon host used as the local DB host fallback | no | @@ -141,7 +141,14 @@ existing SQL or creates an export manifest. with the same flag. A real version/tag mismatch still suggests `supabase stop --all --no-backup` then `supabase start`. -### Legacy shadow baseline cache (`SUPABASE_SHADOW_CACHE`, default ON) +### Shadow baseline cache (`SUPABASE_SHADOW_CACHE`, default ON) + +Stack mode stores cache-eligible baselines at +the managed snapshot key `` and retains three entries by LRU, refreshing mtime on warm +restores. The key includes resolved stack artifacts, runtime, +platform, enabled catalog services, settings, credentials, overlay inputs, and `roles.sql`. +`--no-cache` and a falsy `SUPABASE_SHADOW_CACHE` bypass restore and publication. A failed warm +restore recreates the database; failed publication warns and keeps the live shadow usable. The bundled (pg-delta next) engine provisions both plan shadows through `acquireShadowDatabase` (`pgdelta-next-shadow.layer.ts`): on by default, off when diff --git a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts index 8b1740bdba..bc53b7a157 100644 --- a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts @@ -101,7 +101,7 @@ function syncStackApi(workdir: string, port: number) { destroy: unusedSync, prepare: unusedSync, credentials: unusedSyncFn, - exportSnapshot: unusedSyncFn, + saveSnapshot: unusedSyncFn, restoreSnapshot: unusedSyncFn, resetData: unusedSync, logs: Stream.empty, diff --git a/apps/cli/src/commands/db/shared/pgdelta-next-shadow.layer.ts b/apps/cli/src/commands/db/shared/pgdelta-next-shadow.layer.ts index 6e08a7823a..dfd45f00a1 100644 --- a/apps/cli/src/commands/db/shared/pgdelta-next-shadow.layer.ts +++ b/apps/cli/src/commands/db/shared/pgdelta-next-shadow.layer.ts @@ -293,10 +293,10 @@ export const pgDeltaNextShadowLayer = Layer.effect( }).pipe(Effect.provide(runtimeWith(outputService)), Effect.mapError(nextShadowError)); const stackAcquire = (input: NativeShadowInput, opts: ShadowCacheOpts) => - stackAcquireShadowDatabase( - input.base, - opts.webhooks === undefined ? {} : { webhooks: opts.webhooks }, - ); + stackAcquireShadowDatabase(input.base, { + ...(opts.webhooks === undefined ? {} : { webhooks: opts.webhooks }), + ...(opts.bypassCache === true ? { bypassCache: true } : {}), + }); const stackProvisionMigrations = (input: NativeShadowInput, opts: ShadowCacheOpts) => Effect.gen(function* () { @@ -304,7 +304,7 @@ export const pgDeltaNextShadowLayer = Layer.effect( yield* Effect.scoped(stackMigrateShadow(handle, input.base)); return { migrationsUrl: handle.url, - snapshotKey: undefined, + snapshotKey: handle.snapshotKey, } satisfies ProvisionedMigrationsShadow; }).pipe(Effect.provide(runtime), Effect.mapError(nextShadowError)); @@ -313,8 +313,8 @@ export const pgDeltaNextShadowLayer = Layer.effect( const handle = yield* stackAcquire(input, opts); return { declarativeUrl: handle.url, - restoredFromPgDataSnapshot: false, - snapshotKey: undefined, + restoredFromPgDataSnapshot: handle.restoredFromSnapshot, + snapshotKey: handle.snapshotKey, } satisfies ProvisionedDeclarativeShadow; }).pipe(Effect.provide(runtime), Effect.mapError(nextShadowError)); diff --git a/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts b/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts index ff51128750..f7c433462d 100644 --- a/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts +++ b/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts @@ -135,8 +135,9 @@ describe("pg-delta next stack shadow provisioning", () => { context: { projectId: "test", cwd: root, denoVersion: 2, projectEnv: {} }, toml, projectRef: "test", - bypassCache: true, + bypassCache: false, }); + expect(plan.allowSameDatabaseIdentity).toBe(true); expect(plan.migrationsUrl).not.toBe(plan.declarativeUrl); expect( yield* query(db, plan.migrationsUrl, "SELECT value FROM public.pgdelta_next_probe"), diff --git a/apps/cli/src/commands/db/start/start.integration.test.ts b/apps/cli/src/commands/db/start/start.integration.test.ts index 79efc86553..cb210dfe67 100644 --- a/apps/cli/src/commands/db/start/start.integration.test.ts +++ b/apps/cli/src/commands/db/start/start.integration.test.ts @@ -1614,7 +1614,7 @@ describe("db start stack backend", () => { state.credentialsCalled = true; return { databaseUrl: "postgresql://postgres:secret@127.0.0.1:54329/postgres" }; }), - exportSnapshot: () => Effect.die("unused"), + saveSnapshot: () => Effect.die("unused"), restoreSnapshot: () => Effect.die("unused"), resetData: Effect.die("unused"), }); diff --git a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts index 329bc9bbba..1a106ae502 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts @@ -88,7 +88,7 @@ const makeFixture = (root: string, failPreparation = false) => { followStatus: Stream.empty, logs: Stream.empty, credentials: () => Effect.succeed({}), - exportSnapshot: () => Effect.die("unused"), + saveSnapshot: () => Effect.die("unused"), restoreSnapshot: () => Effect.die("unused"), resetData: Effect.die("unused"), }; diff --git a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts index 1b8272acd2..e3058e8c39 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts @@ -87,7 +87,7 @@ const instance = ( service: "database", credentials: () => Effect.succeed({ databaseUrl: "postgresql://postgres:postgres@127.0.0.1:5432/postgres" }), - exportSnapshot: () => Effect.die("unused"), + saveSnapshot: () => Effect.die("unused"), restoreSnapshot: () => Effect.die("unused"), resetData: Effect.die("unused"), }; diff --git a/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts b/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts index d84606cd98..fde30eeda2 100644 --- a/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts @@ -91,7 +91,7 @@ const makeService = (input: { input.rejectCredentials ? Effect.die("credentials must not run") : Effect.succeed(input.credentials ?? {}), - exportSnapshot: () => Effect.die("unused"), + saveSnapshot: () => Effect.die("unused"), restoreSnapshot: () => Effect.die("unused"), resetData: Effect.die("unused"), }); diff --git a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts index 5e66b5f4da..e5ee61724e 100644 --- a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts +++ b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts @@ -140,7 +140,7 @@ const fixture = ( destroy: Effect.void, prepare: Effect.void, resetData: Effect.void, - exportSnapshot: () => Effect.die("unused"), + saveSnapshot: () => Effect.die("unused"), restoreSnapshot: () => Effect.die("unused"), followStatus: Stream.never, logs: Stream.never, diff --git a/apps/cli/src/commands/migration/squash/SIDE_EFFECTS.md b/apps/cli/src/commands/migration/squash/SIDE_EFFECTS.md index 6f6e176dcf..63c1ff31e0 100644 --- a/apps/cli/src/commands/migration/squash/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/migration/squash/SIDE_EFFECTS.md @@ -9,8 +9,8 @@ migration-history table to match. When `[experimental].stack` is on, each shadow is a fresh database in an invocation-owned, unique temporary stack namespace. The command applies the catalog and project migrations as needed, -then destroys its namespace when the Effect scope closes. Shadow baselines are not cached; -the snapshot cache described below belongs to the legacy container backend. Native artifacts +then destroys its namespace when the Effect scope closes. Stack shadows use the stack baseline +cache described below. Native artifacts are shared through `$SUPABASE_HOME/cache/stack`; shadow state and data use the normal stack registry, so `stack list` and `stack destroy` can find a shadow left by an abrupt CLI exit. Each shadow owns a unique temporary project root and uses an automatically assigned port; `db.shadow_port` applies only to the legacy backend. Schema dumps use the namespace's catalog `pg_dump` tool and runtime database address. @@ -24,6 +24,7 @@ and uses an automatically assigned port; `db.shadow_port` applies only to the le | `/supabase/migrations/` | directory | always | | `/supabase/migrations/_*.sql` | SQL | each migration up to the target, applied to the shadow; the target file's own final content is read by `--version`/baseline lookups | | `/supabase/roles.sql` | SQL | shadow `SetupDatabase` (custom-roles seed); missing file tolerated | +| managed snapshot key `` | backend | warm stack-shadow baseline; managed retention may evict entries and snapshots survive stack destruction | | `/supabase/.env`, `.env.local`, `SUPABASE_ENV`-selected dotenv | dotenv | always (`--yes`/registry/network-id overrides) | | `/supabase/.temp/{project-ref,postgres-version,pooler-url}` | plain text | `--linked` / linked path — skipped when `--project-ref` (or `SUPABASE_PROJECT_ID`) is set | | `~/.supabase/access-token` | plain text | `--linked` without `--password`/`SUPABASE_ACCESS_TOKEN` | @@ -37,8 +38,17 @@ and uses an automatically assigned port; `db.shadow_port` applies only to the le | `/supabase/migrations/.sql` (×N) | — | **deleted** — every earlier merged migration; a per-file failure is non-fatal (printed, not raised) | | scoped temp dir | SQL | shadow's `initSchema`/`ApplyApiPrivileges` SQL (PG≤14) — removed when the scope closes | | `/supabase/.temp/linked-project.json` | JSON | `--linked` (post-run cache, even when the command itself fails) | +| managed snapshot key `` | backend | stack-mode cold shadow baseline; managed retention may evict entries and snapshots survive stack destruction | | `~/.supabase/telemetry.json` | JSON | every invocation (post-run) | +### Stack shadow baseline cache + +Stack mode uses the managed snapshot backend by default and keys each +baseline from the resolved stack artifacts, runtime/platform, enabled catalog services, settings, +credentials, overlay inputs, and `roles.sql`. Set `SUPABASE_SHADOW_CACHE` to a falsy value to opt +out. Warm restore failures recreate the shadow database; failed publication warns and leaves the +live shadow available. + ## Docker - Network ensure (`ensureNetwork`, same as `db diff`/`db pull`). @@ -75,7 +85,7 @@ and uses an automatically assigned port; `db.shadow_port` applies only to the le `SUPABASE_YES`, `DB_PASSWORD`, `SUPABASE_ACCESS_TOKEN`, `SUPABASE_SERVICES_HOSTNAME`, `DOCKER_HOST`/`DOCKER_CONTEXT`/`DOCKER_CONFIG`, `SUPABASE_NETWORK_ID`, `SUPABASE_INTERNAL_IMAGE_REGISTRY`, `SUPABASE_USE_SLIM_IMAGES` (current-pin shadow Postgres and PG15+ realtime/storage/auth migrate-job images → slim `ghcr.io/supabase/cli`; historical pins, PG14, OrioleDB, flag-off `15.8.1.085` stay on docker.io), `SUPABASE_PROJECT_ID`, `SUPABASE_DEBUG`, -`SUPABASE_EXPERIMENTAL`. +`SUPABASE_EXPERIMENTAL`, `SUPABASE_SHADOW_CACHE` (stack shadow baseline cache; on by default, falsy disables restore and publication). ## Exit Codes diff --git a/apps/cli/tests/helpers/storage.ts b/apps/cli/tests/helpers/storage.ts index 6db2d9d2e9..192d3e4843 100644 --- a/apps/cli/tests/helpers/storage.ts +++ b/apps/cli/tests/helpers/storage.ts @@ -214,7 +214,7 @@ export function buildStorageStackApi( ); const database = { ...instance("database-id", databaseCreation, Effect.succeed(dbObservation)), - exportSnapshot: () => Effect.die("unused"), + saveSnapshot: () => Effect.die("unused"), restoreSnapshot: () => Effect.die("unused"), resetData: Effect.die("unused"), } satisfies DatabaseInstance; diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 63da409ade..3c3287dbd0 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -433,7 +433,7 @@ Keep Effect RPC as the transport initially. Its handlers should mostly delegate | Registry | `services.create`, `services.get`, `services.list` | Create and locate instances | | Instance lifecycle | `service.prepare`, `service.start`, `service.stop`, `service.restart`, `service.destroy` | Operate one instance | | Instance observation | `service.status`, `service.ready`, `service.followStatus`, `service.logs`, `service.followLogs`, `service.credentials` | Read state, await health and inspect outputs | -| Database snapshots | `database.exportSnapshot`, `database.restoreSnapshot` | Database-specific storage operations | +| Database snapshots | `database.saveSnapshot`, `database.restoreSnapshot` | Database-specific managed storage operations | | Composition | `composition.configure`, `composition.describe`, `composition.start`, `composition.stop`, `composition.restart` | Define and operate the application selection and dependencies | | Tools | `tools.run`, `tools.writeStdin`, `tools.closeStdin` | Execute an attached command with streamed input/output | @@ -534,7 +534,7 @@ Some foundations are not optional, but they do not need to dominate the service | Ownership | One detached StackHost and exclusive lease; client handles do not own service lifetime | | Ports | Persist public port assignments across stop/start; reuse the same ports, including initially automatic assignments; never silently relocate | | Data | Stop retains data; destroy removes only proven owned data | -| Snapshots | DB-specific stopped export, compatible empty-target restore, complete archive publication and credential reconciliation | +| Snapshots | DB-specific keyed save, compatible empty-target restore, complete entry publication and credential reconciliation | ### Port allocation and endpoint ownership @@ -622,39 +622,41 @@ The artifact cache is independent and shared across stacks. Normal stop preserve Snapshots are initially supported only for `database`, including the shadow-baseline cache use case. Start with that concrete case. -Expose `exportSnapshot` and `restoreSnapshot` on `DatabaseInstance` only. The common service handle retains lifecycle, health and logs; REST and other instance types do not expose unsupported snapshot methods. Keep a database-specific snapshot descriptor. No generic snapshot provider registry, mandatory storage interface or whole-composition snapshot is needed until a second concrete use case requires one. +Expose `saveSnapshot` and `restoreSnapshot` on `DatabaseInstance` only. The common service handle retains lifecycle, health and logs; REST and other instance types do not expose unsupported snapshot methods. Callers identify managed snapshots with opaque keys. No generic snapshot provider registry, mandatory storage interface or whole-composition snapshot is needed until a second concrete use case requires one. ```ts interface DatabaseInstance extends ServiceInstance { readonly service: "database"; - exportSnapshot(options: { destination: string }): Promise; - restoreSnapshot(options: { source: string }): Promise; + saveSnapshot(key: string): Promise; + restoreSnapshot(key: string): Promise; } // `baseline` has already been initialized; `shadow` is a fresh instance. await baseline.stop(); -await baseline.exportSnapshot({ destination: archivePath }); +await baseline.saveSnapshot("baseline"); -await shadow.restoreSnapshot({ source: archivePath }); +if (!(await shadow.restoreSnapshot("baseline"))) { + // A miss leaves the empty instance in place; initialize that same instance. +} await shadow.start(); await shadow.ready(); ``` -The database implementation owns the snapshot format, PostgreSQL data selection, compatibility validation, initialization metadata and credential reconciliation. It uses native filesystem operations or container volume/helper operations through the runtime backend. The orchestrator knows only admission, instance ownership and operation settlement; it never needs to understand PostgreSQL archive contents. +The database implementation owns the snapshot format, PostgreSQL data selection, compatibility validation, initialization metadata and credential reconciliation. It uses native filesystem clone/copy operations or container volume/helper operations through the runtime backend. Native entries live below `cacheRoot`; Docker entries share the data volume, in a separate namespace derived from `cacheRoot`. Docker cache reuse requires the same daemon, `stateRoot`, and `cacheRoot`. Each store retains three entries by last use; saving a key replaces the previous complete entry for that key. Cache entries are disposable and do not promise durability across power loss. The orchestrator knows only admission, instance ownership and operation settlement; it never needs to understand PostgreSQL data contents. Keep the contract narrow: -- Export requires confirmed stopped, initialized data. The caller explicitly stops first; snapshotting does not secretly stop dependents or restart services. -- Restore requires a confirmed stopped instance with empty data. Validate archive safety, format, artifact/runtime compatibility and initialization profile before installing restored data. Reject a nonempty target; there is no overwrite option. +- Save requires confirmed stopped, initialized data. The caller explicitly stops first; snapshotting does not secretly stop dependents or restart services. A save publishes only a complete entry and replaces the previous entry for the same key. +- Restore requires a confirmed stopped instance with empty data. Validate format, artifact/runtime compatibility and initialization profile before installing restored data. A missing key returns `false`; a compatible published entry returns `true`; reject a nonempty target rather than overwriting it. - Both operations occupy the instance's existing serial operation gate and leave lifecycle stopped. Queued start, destroy or another storage operation waits for settlement and revalidates. No new lifecycle states are necessary; the observable pending operation identifies snapshot work. An armed wake route is not a substitute for explicit stop. -- Export publishes only a completed archive. Validate and stage restoration before treating the target as usable. Normal failures clean up this invocation's temporary resources and report any cleanup failure; incomplete restoration is never reported as success. Do not add persisted recovery phases or an operation journal. -- Restore transfers compatible database contents, not the source instance's identity, public port claims or composition membership. The target retains its own data location and configuration, with database-specific credentials reconciled before readiness. +- Native snapshots copy or clone the host data; container snapshots copy database data through a managed volume and helper. Docker data normally lives in a managed volume, while existing host data can be retained through the host-backed fallback. The host storage marker detects a missing or mismatched Docker volume; deleting that volume loses its database data. +- Restore transfers compatible database contents, not the source instance's identity, public port claims or composition membership. The target retains its own data location and configuration, with database-specific credentials reconciled before readiness. Snapshots survive destruction of the source instance because their managed storage is separate. -These are physical database snapshots for the cache use case. A `pg_dump` invocation remains an ordinary client tool for logical exports. CLI code owns cache keys, eviction, migrations and the decision to fall back to rebuilding a baseline. The snapshot API does not acquire those policies. +These are physical database snapshots for the cache use case. A `pg_dump` invocation remains an ordinary client tool for logical exports. CLI code owns cache keys, migrations and the decision to fall back to rebuilding a baseline; managed storage owns publication and retention. The snapshot API does not acquire CLI cache policy. ### Resetting database data -`DatabaseInstance.resetData` removes the selected database instance's owned data and initialization metadata while retaining its registration, configuration, composition bindings, and public port assignments. It is database-specific, alongside snapshot export and restore; other service types do not expose a reset operation. +`DatabaseInstance.resetData` removes the selected database instance's owned data and initialization metadata while retaining its registration, configuration, composition bindings, and public port assignments. It is database-specific, alongside snapshot save and restore; other service types do not expose a reset operation. The caller must stop the database with wake disabled first. Reset runs through the same serialized storage-operation gate as snapshots and leaves the database stopped. Deletion uses the database's ownership checks and runtime-specific filesystem handling. The next normal start initializes a fresh PostgreSQL cluster using the retained configuration. Reset does not apply project migrations or seeds, stop other services, or resume the composition; those decisions belong to the CLI. It adds no lifecycle state or persisted recovery phase. @@ -732,6 +734,6 @@ Verify through consumed integration flows: 3. Register two standalone databases, then start/restart the default composition: neither standalone instance is implicitly started or restarted. Namespace stop/destroy still includes both. Two stacks and multiple databases have separate ports/data; a shadow operation does not block Functions restart. Record every public endpoint, stop the stack, start another parallel stack, then reopen/start the original: every original public port is unchanged. Occupying one saved port with an unrelated listener instead produces a conflict without rewriting its assignment. 4. Start a composition containing lazy database and REST instances: startup returns concrete URLs with their proxy listeners bound and their backend processes still absent. Connecting to the returned database/REST URLs launches the appropriate instances without changing the URLs. No public or RPC sleep operation is exposed. Proxy inactivity can trigger internal sleep; public traffic prevents it; sleep retains wake; explicit stop disables wake; prerequisite idle eligibility is rechecked after dependent sleep. 5. Unexpected exit disables wake and triggers ordinary cleanup of the owned runtime before another launch; Functions can explicitly restart afterward. Traffic cannot restart an exited prerequisite. A stale exit cannot damage a replacement. -6. Normal snapshot failures do not publish partial archives or overwrite existing data. Client disconnection leaves admitted snapshot work owned by the live host. Host-crash recovery is not an acceptance requirement; unavailable observations must not be presented as current running/healthy/stopped state. +6. Normal snapshot failures do not publish partial entries or overwrite existing data. Client disconnection leaves admitted snapshot work owned by the live host. Host-crash recovery is not an acceptance requirement; unavailable observations must not be presented as current running/healthy/stopped state. 7. Container-mode service wiring and dumps receive a reachable runtime-facing database URL as plain data. Dumps stream with bounded buffering and exact cleanup; stackless commands create no managed stack. Stopping/destroying REST leaves Auth on the shared API port working; shared port claims survive removal of the final route. 8. Namespace stop settles executing work and leaves no live runtime resources; failed cleanup cannot report successful stop. Disconnecting a client does not stop admitted lifecycle/snapshot operations. Stopping the last individual instance or finishing the last tool does not retire the StackHost; namespace stop/destroy does. diff --git a/packages/stack/README.md b/packages/stack/README.md index 80c6b01f5a..465b0e52e1 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -45,11 +45,11 @@ await stack.tools.run(postgres.psql({ major: 17 }), { }); await database.stop(); -await database.exportSnapshot("/tmp/example-stack/baseline.tar"); +await database.saveSnapshot("baseline"); await stack.close(); // disconnects this client ``` -`start` and `ready` are separate operations. `restart({ config })` replaces recipe configuration while retaining the instance identity and endpoint intentions. A health failure leaves a launched process running and observable; it does not prevent `stop`. Database snapshots require a stopped instance with wake disabled. Other service handles have no snapshot methods. +`start` and `ready` are separate operations. `restart({ config })` replaces recipe configuration while retaining the instance identity and endpoint intentions. A health failure leaves a launched process running and observable; it does not prevent `stop`. Database snapshots require a stopped instance with wake disabled. `saveSnapshot(key)` publishes complete data to managed backend storage and replaces the previous entry for that key; `restoreSnapshot(key)` returns `false` on a miss and `true` after restoring a compatible entry. Managed retention may evict older keys, while snapshots survive destruction of the source stack. Other service handles have no snapshot methods. Creating a service records its definition. Configured public ports are bound during startup and retained across normal stop/start and owner reopening. An occupied saved port reports a conflict instead of moving. Omitted public endpoints are not exposed. @@ -61,7 +61,7 @@ On Linux, native Functions project files must be outside `/tmp`: Edge Runtime us `open({ id, stateRoot, cacheRoot })` reconnects to a saved stack. The package stores the stack document at `//state.json` and service data at `//data/`. `discover({ stateRoot })` lists saved definitions and port assignments separately from live-owner availability. Offline definitions are not live lifecycle observations. -The stack owns database, Functions bootstrap, and tool-job directories below its data directory. Storage uploads remain at the caller-supplied Storage `filePath` and are preserved when the stack is destroyed; the caller owns that directory. +The stack owns database, Functions bootstrap, and tool-job directories below its data directory. Storage uploads remain at the caller-supplied Storage `filePath` and are preserved when the stack is destroyed; the caller owns that directory. Host metadata remains under `stateRoot`; native database data uses host files. Docker database data normally uses a managed volume, while existing host data is retained through the host-backed fallback. A host marker records the selected Docker storage and detects a missing or mismatched volume; deleting that volume loses the associated database data. Native snapshot entries live below `cacheRoot`. Docker snapshots share the managed data volume in a separate namespace derived from `cacheRoot`, so they survive source destruction and can use filesystem cloning. A Docker cache hit requires the same daemon, `stateRoot`, and `cacheRoot`. There is no portable tar snapshot API. Without an explicit database root key, PostgreSQL generates one in its data directory. The key survives stop/reopen and is removed with the database data on reset or destroy. diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index c459ec053d..8047ed1760 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -49,7 +49,6 @@ import { type CatalogRecipe, serviceSchemas, } from "./services/Catalog.ts"; -import { makeDatabaseSnapshots, type DatabaseSnapshot } from "./services/DatabaseSnapshot.ts"; import * as State from "./State.ts"; import type { SavedInstance, SavedStack } from "./State.ts"; @@ -113,14 +112,8 @@ export interface Interface { from: "host" | "runtime", ) => Effect.Effect>, OwnerError>; readonly snapshots: { - readonly exportSnapshot: ( - id: string, - destination: string, - ) => Effect.Effect; - readonly restoreSnapshot: ( - id: string, - source: string, - ) => Effect.Effect; + readonly saveSnapshot: (id: string, key: string) => Effect.Effect; + readonly restoreSnapshot: (id: string, key: string) => Effect.Effect; readonly resetData: (id: string) => Effect.Effect; }; readonly namespace: { @@ -670,27 +663,6 @@ const makeOwnerWithDependencies = ( .pipe(Effect.mapError((cause) => errorFor("storage", cause))); }), ); - const snapshotStore = (id: string) => - get(id).pipe( - Effect.flatMap(({ creation }) => - creation.service === "database" - ? makeDatabaseSnapshots({ - instanceRoot: path.join(options.root, id), - runtime, - version: creation.config.version, - stackId: options.saved.id, - instanceId: id, - }).pipe( - Effect.provideService(FileSystem.FileSystem, fs), - Effect.provideService(Path.Path, path), - Effect.provideService(Crypto.Crypto, crypto), - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - Effect.mapError((cause) => errorFor("snapshot", cause)), - ) - : Effect.fail(errorFor("snapshot", "Snapshots are only supported for databases")), - ), - ); - const createService = Effect.fn("Owner.createService")(function* (input: unknown) { yield* Ref.get(draining).pipe( Effect.flatMap((isDraining) => @@ -892,27 +864,55 @@ const makeOwnerWithDependencies = ( Effect.flatMap((values) => Effect.forEach(values, (value) => observation(value.id))), Effect.withSpan("Owner.restartComposition"), ); - const exportSnapshot = Effect.fn("Owner.exportSnapshot")((id: string, destination: string) => - snapshotStore(id).pipe( - Effect.flatMap((store) => - storage( - id, - store - .exportSnapshot({ destination }) - .pipe(Effect.mapError((cause) => errorFor("snapshot", cause))), - ), + const saveSnapshot = Effect.fn("Owner.saveSnapshot")((id: string, key: string) => + get(id).pipe( + Effect.flatMap(({ creation }) => + creation.service === "database" + ? storage( + id, + Effect.gen(function* () { + const current = yield* getRecipe(id); + const save = current.saveDatabaseSnapshot; + if (save === undefined) + return yield* errorFor("saveSnapshot", "Database snapshots are unavailable"); + return yield* Effect.acquireUseRelease( + Scope.make("parallel"), + (scope) => + save({ id, config: current.creation, scope }, key).pipe( + Effect.mapError((cause) => errorFor("saveSnapshot", cause)), + ), + (scope, exit) => Scope.close(scope, exit), + ); + }), + ) + : Effect.fail(errorFor("saveSnapshot", "Snapshots are only supported for databases")), ), ), ); - const restoreSnapshot = Effect.fn("Owner.restoreSnapshot")((id: string, source: string) => - snapshotStore(id).pipe( - Effect.flatMap((store) => - storage( - id, - store - .restoreSnapshot({ source }) - .pipe(Effect.mapError((cause) => errorFor("snapshot", cause))), - ), + const restoreSnapshot = Effect.fn("Owner.restoreSnapshot")((id: string, key: string) => + get(id).pipe( + Effect.flatMap(({ creation }) => + creation.service === "database" + ? storage( + id, + Effect.gen(function* () { + const current = yield* getRecipe(id); + const restore = current.restoreDatabaseSnapshot; + if (restore === undefined) + return yield* errorFor("restoreSnapshot", "Database snapshots are unavailable"); + return yield* Effect.acquireUseRelease( + Scope.make("parallel"), + (scope) => + restore({ id, config: current.creation, scope }, key).pipe( + Effect.mapError((cause) => errorFor("restoreSnapshot", cause)), + ), + (scope, exit) => Scope.close(scope, exit), + ); + }), + ) + : Effect.fail( + errorFor("restoreSnapshot", "Snapshots are only supported for databases"), + ), ), ), ); @@ -1003,7 +1003,7 @@ const makeOwnerWithDependencies = ( }, credentials, snapshots: { - exportSnapshot, + saveSnapshot, restoreSnapshot, resetData, }, diff --git a/packages/stack/src/Rpc.ts b/packages/stack/src/Rpc.ts index f8186ff8d8..9f6ac6fa38 100644 --- a/packages/stack/src/Rpc.ts +++ b/packages/stack/src/Rpc.ts @@ -2,7 +2,6 @@ import { Data, Schema } from "effect"; import { Rpc, RpcGroup } from "effect/unstable/rpc"; import { ServiceCreation } from "./services/Catalog.ts"; import { CompositionConfig } from "./Orchestrator.ts"; -import { SnapshotDescriptor } from "./services/DatabaseSnapshot.ts"; import { PgProveOptions, PostgresTool } from "./Tools.ts"; const Outcome = Schema.Struct({ @@ -56,7 +55,6 @@ export const Definition = Schema.Struct({ id: Schema.String, creation: ServiceCr export interface Definition extends Schema.Schema.Type {} const Instance = { id: Schema.String }; -const Snapshot = Schema.Struct({ descriptor: SnapshotDescriptor, destination: Schema.String }); const Log = Schema.Struct({ stream: Schema.Literals(["stdout", "stderr"]), bytes: Schema.Uint8ArrayFromBase64, @@ -100,14 +98,13 @@ export const StackRpc = RpcGroup.make( success: Schema.Record(Schema.String, Schema.String), error: StackErrorSchema, }), - Rpc.make("exportSnapshot", { - payload: { ...Instance, destination: Schema.String }, - success: Snapshot, + Rpc.make("saveSnapshot", { + payload: { ...Instance, key: Schema.String }, error: StackErrorSchema, }), Rpc.make("restoreSnapshot", { - payload: { ...Instance, source: Schema.String }, - success: Snapshot, + payload: { ...Instance, key: Schema.String }, + success: Schema.Boolean, error: StackErrorSchema, }), Rpc.make("resetData", { payload: Instance, error: StackErrorSchema }), diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index 4fc986e0d8..946c158666 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -251,19 +251,13 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( owner .credentials(id, from) .pipe(Effect.mapError((cause) => stackError("credentials", cause))), - exportSnapshot: ({ - id, - destination, - }: { - readonly id: string; - readonly destination: string; - }) => + saveSnapshot: ({ id, key }: { readonly id: string; readonly key: string }) => owner.snapshots - .exportSnapshot(id, destination) - .pipe(Effect.mapError((cause) => stackError("exportSnapshot", cause))), - restoreSnapshot: ({ id, source }: { readonly id: string; readonly source: string }) => + .saveSnapshot(id, key) + .pipe(Effect.mapError((cause) => stackError("saveSnapshot", cause))), + restoreSnapshot: ({ id, key }: { readonly id: string; readonly key: string }) => owner.snapshots - .restoreSnapshot(id, source) + .restoreSnapshot(id, key) .pipe(Effect.mapError((cause) => stackError("restoreSnapshot", cause))), resetData: ({ id }: { readonly id: string }) => owner.snapshots diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index 54a602ff25..54c14dfb57 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -16,7 +16,6 @@ import { type Observation, } from "./Rpc.ts"; import { ServiceCreation } from "./services/Catalog.ts"; -import type { DatabaseSnapshot } from "./services/DatabaseSnapshot.ts"; import * as Orchestrator from "./Orchestrator.ts"; import type { PgProveOptions, PostgresTool } from "./Tools.ts"; @@ -26,7 +25,6 @@ export type { ServiceCreation } from "./services/Catalog.ts"; export type { CompositionConfig } from "./Orchestrator.ts"; export type { SupabaseCompositionOptions } from "./composition/Supabase.ts"; export type { Observation } from "./Rpc.ts"; -export type { DatabaseSnapshot } from "./services/DatabaseSnapshot.ts"; export type { PgProveOptions } from "./Tools.ts"; const stateFor = (root: string) => State.Service.pipe(Effect.provide(State.layer({ root }))); @@ -84,8 +82,8 @@ export interface ServiceInstance { } /** A database instance with stopped-data snapshot operations. */ export interface DatabaseInstance extends ServiceInstance<"database"> { - readonly exportSnapshot: (destination: string) => Effect.Effect; - readonly restoreSnapshot: (source: string) => Effect.Effect; + readonly saveSnapshot: (key: string) => Effect.Effect; + readonly restoreSnapshot: (key: string) => Effect.Effect; /** Removes database-owned data while preserving the instance registration. */ readonly resetData: Effect.Effect; } @@ -228,10 +226,9 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( case "database": return { ...common(id, "database"), - exportSnapshot: (destination) => - call("exportSnapshot", (rpc) => rpc.exportSnapshot({ id, destination })), - restoreSnapshot: (source) => - call("restoreSnapshot", (rpc) => rpc.restoreSnapshot({ id, source })), + saveSnapshot: (key) => call("saveSnapshot", (rpc) => rpc.saveSnapshot({ id, key })), + restoreSnapshot: (key) => + call("restoreSnapshot", (rpc) => rpc.restoreSnapshot({ id, key })), resetData: call("resetData", (rpc) => rpc.resetData({ id })), }; case "rest": diff --git a/packages/stack/src/index.ts b/packages/stack/src/index.ts index ace2e539a7..7ecb2b0983 100644 --- a/packages/stack/src/index.ts +++ b/packages/stack/src/index.ts @@ -31,7 +31,6 @@ const decodeCreation = (creation: ServiceCreation) => ); export type { CompositionConfig } from "./Orchestrator.ts"; export type { Observation } from "./Rpc.ts"; -export type { DatabaseSnapshot } from "./services/DatabaseSnapshot.ts"; export type { PgProveOptions } from "./effect.ts"; export type { SupabaseCompositionOptions } from "./effect.ts"; export type { CreateOptions, OpenOptions, StackLocations } from "./effect.ts"; @@ -70,14 +69,8 @@ export interface ServiceInstance { } /** Database storage operations require a stopped instance with wake disabled. */ export interface DatabaseInstance extends ServiceInstance<"database"> { - readonly exportSnapshot: ( - destination: string, - options?: CallOptions, - ) => Promise; - readonly restoreSnapshot: ( - source: string, - options?: CallOptions, - ) => Promise; + readonly saveSnapshot: (key: string, options?: CallOptions) => Promise; + readonly restoreSnapshot: (key: string, options?: CallOptions) => Promise; /** Removes database-owned data while preserving the instance registration. */ readonly resetData: (options?: CallOptions) => Promise; } @@ -206,9 +199,8 @@ const adapt = (handle: StackEffect.Stack, runtime: Runtime) => { case "database": return { ...common(service), - exportSnapshot: (destination, options) => - run(service.exportSnapshot(destination), options), - restoreSnapshot: (source, options) => run(service.restoreSnapshot(source), options), + saveSnapshot: (key, options) => run(service.saveSnapshot(key), options), + restoreSnapshot: (key, options) => run(service.restoreSnapshot(key), options), resetData: (options) => run(service.resetData, options), }; case "rest": diff --git a/packages/stack/src/public.e2e.test.ts b/packages/stack/src/public.e2e.test.ts index b97c803e66..31abce8023 100644 --- a/packages/stack/src/public.e2e.test.ts +++ b/packages/stack/src/public.e2e.test.ts @@ -277,8 +277,9 @@ it.live( ); expect(seed.exitCode).toBe(0); yield* Effect.tryPromise(() => database.stop()); - const archive = path.join(root, "public-snapshot.tar"); - yield* Effect.tryPromise(() => database.exportSnapshot(archive)); + const snapshotKey = "public-roundtrip"; + yield* Effect.tryPromise(() => database.saveSnapshot(snapshotKey)); + yield* Effect.tryPromise(() => database.destroy()); const restored = yield* Effect.tryPromise(() => client.services.create({ service: "database", @@ -291,7 +292,9 @@ it.live( endpoints: { sql: { port: "auto" } }, }), ); - yield* Effect.tryPromise(() => restored.restoreSnapshot(archive)); + expect(yield* Effect.tryPromise(() => restored.restoreSnapshot(snapshotKey))).toBe( + true, + ); yield* Effect.tryPromise(() => restored.start()); yield* Effect.tryPromise(() => restored.ready()); const restoredCredentials = yield* Effect.tryPromise(() => @@ -318,7 +321,7 @@ it.live( .trim(), ).toBe("roundtrip"); expect((yield* stack.services.list).map((service) => service.id)).toContain( - database.id, + restored.id, ); const selected = yield* Effect.tryPromise(() => client.composition.supabase([ diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index 82bed8b63a..82c62475b0 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -34,6 +34,8 @@ interface ContainerSpec { readonly source: string; readonly target: string; readonly readOnly: boolean; + readonly type?: "bind" | "volume"; + readonly volumeSubpath?: string; }>; readonly workingDir?: string; readonly ports?: ReadonlyArray; @@ -143,8 +145,14 @@ export const makeContainerRuntime = (options: { }); const prepare = Effect.fn("Container.prepare")(function* (image: string) { - const present = yield* run(["image", "ls", "--quiet", "--no-trunc", image]); - if (present.length === 0) yield* run(["pull", image], { timeout: "5 minutes" }); + const inspected = yield* run(["image", "inspect", "--format", "{{.Id}}", image]).pipe( + Effect.catchTag("ContainerError", (error) => + /no such image|image .*not known/iu.test(error.message) + ? Effect.succeed("") + : Effect.fail(error), + ), + ); + if (inspected.length === 0) yield* run(["pull", image], { timeout: "5 minutes" }); }); const launch = Effect.fn("Container.launch")(function* ( @@ -201,7 +209,15 @@ export const makeContainerRuntime = (options: { envPath, ...(spec.mounts ?? []).flatMap((mount) => [ "--mount", - `type=bind,${mountField("src", mount.source)},${mountField("dst", mount.target)}${mount.readOnly ? ",ro" : ""}`, + [ + `type=${mount.type ?? "bind"}`, + mountField("src", mount.source), + mountField("dst", mount.target), + ...(mount.volumeSubpath === undefined + ? [] + : [mountField("volume-subpath", mount.volumeSubpath)]), + ...(mount.readOnly ? ["ro"] : []), + ].join(","), ]), ...(spec.workingDir === undefined ? [] : ["--workdir", spec.workingDir]), ...(spec.ports ?? []).flatMap((port) => ["--publish", `127.0.0.1::${port}`]), diff --git a/packages/stack/src/services/Catalog.ts b/packages/stack/src/services/Catalog.ts index a7823c4266..d300e627d3 100644 --- a/packages/stack/src/services/Catalog.ts +++ b/packages/stack/src/services/Catalog.ts @@ -187,6 +187,35 @@ const databaseRecipe = ( : Effect.fail( new CatalogError({ operation: "reset", message: "Service kind cannot change" }), ), + saveDatabaseSnapshot: (context, key) => + context.config.service === "database" + ? component + .saveSnapshot({ ...context, config: context.config.config }, key) + .pipe( + Effect.mapError( + (cause) => + new CatalogError({ operation: "snapshot-save", message: cause.message, cause }), + ), + ) + : Effect.fail( + new CatalogError({ operation: "snapshot-save", message: "Service kind cannot change" }), + ), + restoreDatabaseSnapshot: (context, key) => + context.config.service === "database" + ? component + .restoreSnapshot({ ...context, config: context.config.config }, key) + .pipe( + Effect.mapError( + (cause) => + new CatalogError({ operation: "snapshot-restore", message: cause.message, cause }), + ), + ) + : Effect.fail( + new CatalogError({ + operation: "snapshot-restore", + message: "Service kind cannot change", + }), + ), endpoint: (name) => name === "sql" ? component.endpoint.pipe( @@ -343,4 +372,12 @@ export type CatalogRecipe = RecipeCatalogRecipe & { readonly resetDatabaseData?: ( context: ServiceInstanceContext, ) => Effect.Effect; + readonly saveDatabaseSnapshot?: ( + context: ServiceInstanceContext, + key: string, + ) => Effect.Effect; + readonly restoreDatabaseSnapshot?: ( + context: ServiceInstanceContext, + key: string, + ) => Effect.Effect; }; diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index 620d0ee525..da4ed5591a 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -49,6 +49,11 @@ import { } from "../runtime/NativeProcess.ts"; import type { StackId } from "../identity/StackId.ts"; import { EndpointIntent, serviceCreation } from "./Recipe.ts"; +import { makeDatabaseSnapshots } from "./DatabaseSnapshot.ts"; +import { + makeDockerDatabaseStorage, + type DockerDatabaseStorage, +} from "../storage/DockerDatabaseStorage.ts"; export const DatabaseConfig = Schema.Struct({ version: Schema.String, @@ -105,6 +110,14 @@ export interface DatabaseComponent { readonly resetData: ( context: ServiceInstanceContext, ) => Effect.Effect; + readonly saveSnapshot: ( + context: ServiceInstanceContext, + key: string, + ) => Effect.Effect; + readonly restoreSnapshot: ( + context: ServiceInstanceContext, + key: string, + ) => Effect.Effect; readonly endpoint: Effect.Effect; readonly logs: Stream.Stream; } @@ -203,6 +216,7 @@ const health = Effect.fn("Database.health")(( readonly instanceRoot: string; readonly version: string; readonly runtime: DatabaseRuntime; + readonly markInitialized?: Effect.Effect; }, ): Effect.Effect => { const host = endpoint.kind === "unix" ? endpoint.path : endpoint.host; @@ -281,6 +295,7 @@ const health = Effect.fn("Database.health")(( prefix: ".ready-", }); yield* context.fs.writeFileString(`${stage}/marker`, readyMarker, { mode: 0o600 }); + if (context.markInitialized !== undefined) yield* context.markInitialized; yield* context.fs.rename( `${stage}/marker`, `${context.instanceRoot}/.supabase-database-ready.json`, @@ -467,6 +482,37 @@ export const makeDatabase = ( options.runtime === "native" ? undefined : yield* makeContainerRuntime({ engine: options.runtime }); + const storage: DockerDatabaseStorage | undefined = + options.runtime === "native" + ? undefined + : yield* makeDockerDatabaseStorage({ + runtime: options.runtime, + stackId: String(options.stackId), + instanceId: options.instanceId, + instanceRoot, + root: options.root, + cacheRoot: options.cacheRoot, + fs, + path, + crypto, + container, + spawner, + }).pipe(Effect.mapError((cause) => databaseError("storage", cause))); + + const snapshots = (version: string) => + makeDatabaseSnapshots({ + instanceRoot, + cacheRoot: options.cacheRoot, + runtime: options.runtime, + version, + stackId: String(options.stackId), + instanceId: options.instanceId, + }).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(Crypto.Crypto, crypto), + Effect.mapError((cause) => errorFor("snapshot", cause)), + ); const dataCommand = Effect.fn("Database.containerFiles")( function* (version: string, args: ReadonlyArray) { @@ -484,13 +530,20 @@ export const makeDatabase = ( entrypoint: "/usr/bin/busybox", args, env: {}, - mounts: [{ source: instanceRoot, target: "/instance", readOnly: false }], + mounts: [ + storage === undefined + ? { source: instanceRoot, target: "/instance", readOnly: false } + : yield* storage.mount(version).pipe( + Effect.map((mount) => ({ ...mount, target: "/var/lib/postgresql/data" })), + Effect.mapError((cause) => errorFor("data", cause)), + ), + ], }); return yield* Effect.acquireUseRelease( Effect.succeed(child), (child) => Effect.gen(function* () { - const [, stdout, , code] = yield* Effect.all( + const [, stdout, stderr, code] = yield* Effect.all( [ Stream.empty.pipe(Stream.run(child.stdin)), child.stdout.pipe( @@ -500,13 +553,22 @@ export const makeDatabase = ( (text, chunk) => (text + chunk).slice(-65536), ), ), - child.stderr.pipe(Stream.runDrain), + child.stderr.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (text, chunk) => (text + chunk).slice(-4096), + ), + ), child.exitCode, ], { concurrency: "unbounded" }, ); if (code !== 0) - return yield* errorFor("data", "Container filesystem operation failed"); + return yield* errorFor( + "data", + stderr.trim() || "Container filesystem operation failed", + ); return stdout; }), (child) => child.stop.pipe(Effect.andThen(child.remove)), @@ -570,6 +632,7 @@ export const makeDatabase = ( const prepare = Effect.fn("Database.prepare")( function* (input: DatabaseConfig) { + if (storage !== undefined) yield* storage.prepare(postgresVersion(input.version)); const markerPath = path.join(instanceRoot, ".supabase-database-ready.json"); const hasMarker = yield* fs.exists(markerPath); if (hasMarker) { @@ -595,18 +658,6 @@ export const makeDatabase = ( ); } yield* prepareArtifact(input); - if (!hasMarker && container !== undefined) { - const initialized = (yield* dataCommand(input.version, [ - "sh", - "-c", - "if [ -f /instance/data/PG_VERSION ]; then /usr/bin/busybox cat /instance/data/PG_VERSION; fi", - ])).trim(); - if (initialized !== "" && initialized !== postgresVersion(input.version).split(".")[0]) - return yield* errorFor( - "prepare", - "Initialized PostgreSQL major does not match the requested configuration", - ); - } }, Effect.mapError((cause) => errorFor("prepare", cause)), ); @@ -618,9 +669,19 @@ export const makeDatabase = ( Effect.gen(function* () { const config = { ...context.config, version: postgresVersion(context.config.version) }; const dataPath = path.join(instanceRoot, "data"); - yield* fs - .makeDirectory(dataPath, { recursive: true, mode: 0o700 }) - .pipe(Effect.mapError((cause) => errorFor("launch", cause))); + const dataMount = + storage === undefined + ? yield* fs.makeDirectory(dataPath, { recursive: true, mode: 0o700 }).pipe( + Effect.mapError((cause) => errorFor("launch", cause)), + Effect.as({ + source: dataPath, + target: "/var/lib/postgresql/data", + readOnly: false, + }), + ) + : yield* storage + .mount(config.version) + .pipe(Effect.mapError((cause) => errorFor("launch", cause))); const rootKeyPath = config.rootKey === undefined ? undefined : path.join(instanceRoot, "pgsodium_root.key"); if (rootKeyPath !== undefined && config.rootKey !== undefined) @@ -673,6 +734,12 @@ export const makeDatabase = ( instanceRoot, version: config.version, runtime: options.runtime, + markInitialized: + storage === undefined + ? undefined + : storage + .markInitialized(config.version) + .pipe(Effect.mapError((cause) => errorFor("health", cause))), }), exit: processExit(process.exitCode), stop: process.kill.pipe(Effect.mapError((cause) => errorFor("stop", cause))), @@ -689,7 +756,13 @@ export const makeDatabase = ( const selectedContainer = container; if (selectedContainer === undefined) return yield* errorFor("launch", "Container runtime is unavailable"); - yield* dataCommand(config.version, ["chown", "100:101", "/instance/data"]); + if ( + storage === undefined || + (yield* storage.needsDataChown.pipe( + Effect.mapError((cause) => errorFor("launch", cause)), + )) + ) + yield* dataCommand(config.version, ["chown", "100:101", "/var/lib/postgresql/data"]); const launched = yield* selectedContainer .launch({ image: image.image, @@ -707,7 +780,7 @@ export const makeDatabase = ( }, args: ["-p", "5432", "-c", "listen_addresses=*", ...settings], mounts: [ - { source: dataPath, target: "/var/lib/postgresql/data", readOnly: false }, + dataMount, ...(rootKeyPath === undefined ? [] : [ @@ -756,6 +829,12 @@ export const makeDatabase = ( instanceRoot, version: config.version, runtime: options.runtime, + markInitialized: + storage === undefined + ? undefined + : storage + .markInitialized(config.version) + .pipe(Effect.mapError((cause) => errorFor("health", cause))), }, ), remove: runtimeFromContainer(launched).remove.pipe( @@ -776,36 +855,63 @@ export const makeDatabase = ( String(options.stackId), options.instanceId, Effect.gen(function* () { + if (storage !== undefined) { + yield* storage.destroyData(postgresVersion(context.config.version)); + return; + } if (container === undefined || !(yield* fs.exists(path.join(instanceRoot, "data")))) return; - const artifact = yield* resolveArtifact({ - service: "database", - version: postgresVersion(context.config.version), - }); - yield* container.prepare(artifact.image); - yield* dataCommand(context.config.version, ["rm", "-rf", "/instance/data"]); }).pipe(Effect.mapError((cause) => errorFor("destroy", cause))), ), }; const resetData = Effect.fn("Database.resetData")( (context: ServiceInstanceContext) => - definition - .removeData(context) - .pipe( - Effect.andThen( - ensureOwnedRoot( - fs, - path, - instanceRoot, - String(options.stackId), - options.instanceId, - ).pipe(Effect.mapError((cause) => errorFor("reset", cause))), + storage === undefined + ? definition + .removeData(context) + .pipe( + Effect.andThen( + ensureOwnedRoot( + fs, + path, + instanceRoot, + String(options.stackId), + options.instanceId, + ).pipe(Effect.mapError((cause) => errorFor("reset", cause))), + ), + ) + : storage.removeData(postgresVersion(context.config.version)).pipe( + Effect.andThen( + ensureOwnedRoot( + fs, + path, + instanceRoot, + String(options.stackId), + options.instanceId, + ).pipe(Effect.mapError((cause) => errorFor("reset", cause))), + ), + Effect.mapError((cause) => errorFor("reset", cause)), ), - ), ); return { definition, resetData, + saveSnapshot: (context, key) => + storage === undefined + ? Effect.flatMap(snapshots(context.config.version), (store) => + store.saveSnapshot(key), + ).pipe(Effect.mapError((cause) => errorFor("snapshot", cause))) + : storage + .saveSnapshot(postgresVersion(context.config.version), key) + .pipe(Effect.mapError((cause) => errorFor("snapshot", cause))), + restoreSnapshot: (context, key) => + storage === undefined + ? Effect.flatMap(snapshots(context.config.version), (store) => + store.restoreSnapshot(key), + ).pipe(Effect.mapError((cause) => errorFor("snapshot", cause))) + : storage + .restoreSnapshot(postgresVersion(context.config.version), key) + .pipe(Effect.mapError((cause) => errorFor("snapshot", cause))), endpoint: Ref.get(endpoint).pipe( Effect.flatMap((value) => value === undefined diff --git a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts index 328e9925eb..498aabc837 100644 --- a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts +++ b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts @@ -1,349 +1,317 @@ -import { PgClient } from "@effect/sql-pg"; -import { NodeHttpClient, NodeServices } from "@effect/platform-node"; -import { expect, it } from "@effect/vitest"; -import { Context, Data, Effect, FileSystem, Layer, Path, Redacted, Schema, Stream } from "effect"; -import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { makeService, ServiceError } from "../Service.ts"; -import { makeDatabase, type BackendEndpoint, type DatabaseConfig } from "./Database.ts"; -import { SnapshotDescriptor, makeDatabaseSnapshots } from "./DatabaseSnapshot.ts"; -import { cleanupDockerRoot } from "../../tests/docker-cleanup.ts"; +import { NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Clock, Crypto, Effect, Exit, FileSystem, Option, Path, Schema, Scope } from "effect"; +import { makeDatabaseSnapshots } from "./DatabaseSnapshot.ts"; -class SnapshotTestError extends Data.TaggedError("SnapshotTestError")<{ - readonly message: string; -}> {} +const version = "17.6.1.173"; -const config: DatabaseConfig = { - version: "17", - databasePassword: Redacted.make("supabase-snapshot-source-password"), - jwtSecret: Redacted.make("supabase-snapshot-jwt-secret"), - jwtExpiry: 3600, -}; +const setup = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "database-snapshot-" }); + const path = yield* Path.Path; + const cache = path.join(root, "cache"); -const query = (endpoint: BackendEndpoint, password: Redacted.Redacted, statement: string) => - Effect.scoped( + const makeInstance = (name: string) => Effect.gen(function* () { - const host = endpoint.kind === "unix" ? endpoint.path : endpoint.host; - const services = yield* Layer.build( - PgClient.layer({ - host, - port: endpoint.port, - database: "postgres", - username: "supabase_admin", - password, - }), + const instance = path.join(root, name); + yield* fs.makeDirectory(path.join(instance, "data"), { recursive: true }); + yield* fs.writeFileString(path.join(instance, "data", "PG_VERSION"), "17\n"); + yield* fs.writeFileString( + path.join(instance, ".supabase-database-ready.json"), + '{"version":"17.6.1.173","runtime":"native","profile":"supabase"}', ); - return yield* Context.get(services, PgClient.PgClient).unsafe(statement); - }), - ); - -const storageError = (cause: unknown) => - cause instanceof ServiceError - ? cause - : new ServiceError({ - operation: "storage", - message: cause instanceof Error ? cause.message : String(cause), - cause, - }); + return instance; + }); -const Marker = Schema.fromJsonString( - Schema.Struct({ - version: Schema.String, - runtime: Schema.Literals(["native", "docker", "podman"]), - profile: Schema.Literal("supabase"), - }), -); + return { fs, path, root, cache, makeInstance }; +}); -const writeMarker = ( - fs: FileSystem.FileSystem, - root: string, - runtime: "native" | "docker" | "podman", - version = "17.6.1.173", -) => - Effect.gen(function* () { - const marker = yield* Schema.encodeEffect(Marker)({ version, runtime, profile: "supabase" }); - yield* fs.writeFileString(`${root}/.supabase-database-ready.json`, marker, { mode: 0o600 }); +const store = (instance: string, cache: string) => + makeDatabaseSnapshots({ + instanceRoot: instance, + cacheRoot: cache, + runtime: "native", + version, + stackId: "test", + instanceId: "database", }); -const prepareRoot = ( - fs: FileSystem.FileSystem, - runtime: "native" | "docker" | "podman", - prefix: string, -) => +const entries = (fs: FileSystem.FileSystem, path: Path.Path, cache: string) => Effect.gen(function* () { - const root = yield* fs.makeTempDirectoryScoped({ prefix }); - yield* fs.makeDirectory(`${root}/data`, { recursive: true }); - yield* fs.writeFileString(`${root}/data/PG_VERSION`, "17\n"); - yield* writeMarker(fs, root, runtime); - return root; + const root = path.join(cache, "stack-database-snapshots", "entries"); + const names = yield* fs.readDirectory(root); + const result: Array<{ readonly name: string; readonly key: string }> = []; + for (const name of names) { + const descriptor = yield* Schema.decodeEffect( + Schema.fromJsonString(Schema.Struct({ logicalKey: Schema.String })), + )(yield* fs.readFileString(path.join(root, name, "descriptor.json"))).pipe( + Effect.orElseSucceed(() => undefined), + ); + if (descriptor !== undefined) result.push({ name, key: descriptor.logicalKey }); + } + return result; }); -const tar = (args: ReadonlyArray) => - Effect.scoped( - Effect.gen(function* () { - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const child = yield* spawner.spawn(ChildProcess.make("tar", args, { stdin: "ignore" })); - const [stdout, stderr, code] = yield* Effect.all( - [ - child.stdout.pipe(Stream.decodeText, Stream.mkString), - child.stderr.pipe(Stream.decodeText, Stream.mkString), - child.exitCode, - ], - { concurrency: "unbounded" }, - ); - if (Number(code) !== 0) return yield* new SnapshotTestError({ message: stderr }); - return stdout; - }), - ); +const entryPath = (path: Path.Path, cache: string, name: string) => + path.join(cache, "stack-database-snapshots", "entries", name); -for (const runtime of ["native", "docker"] as const) { - it.live( - `exports and restores SQL data through the ${runtime} database service`, - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: `snapshot-${runtime}-` }); - if (runtime === "docker") yield* Effect.addFinalizer(() => cleanupDockerRoot(root)); - const source = yield* makeDatabase({ - stackId: `snapshot-${runtime}`, - instanceId: "source", - root, - cacheRoot: "/tmp/supabase-stack-artifacts", - runtime, - }); - const sourceService = yield* makeService(source.definition, { - id: `database:${runtime}:source`, - config, - }); - yield* sourceService.start; - yield* sourceService.ready; - yield* query( - yield* source.endpoint, - config.databasePassword, - "CREATE TABLE snapshot_data (value text NOT NULL); INSERT INTO snapshot_data VALUES ('restored-row')", - ); - yield* sourceService.stop; +const live = ( + effect: Effect.Effect, +) => Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - const sourceRoot = path.join(root, "source"); - const sourceSnapshots = yield* makeDatabaseSnapshots({ - instanceRoot: sourceRoot, - runtime, - version: config.version, - stackId: `snapshot-${runtime}`, - instanceId: "source", - }); - const archive = path.join(root, `${runtime}.tar`); - yield* sourceService.storage( - sourceSnapshots - .exportSnapshot({ destination: archive }) - .pipe(Effect.mapError(storageError)), - ); +describe("managed native database snapshots", () => { + it.live("saves, restores, replaces, and isolates source and destination mutations", () => + live( + Effect.gen(function* () { + const { fs, path, cache, makeInstance } = yield* setup; + const source = yield* makeInstance("source"); + const target = yield* makeInstance("target"); + yield* fs.writeFileString(path.join(source, "data", "fixture"), "first"); + const sourceStore = yield* store(source, cache); + yield* sourceStore.saveSnapshot("same-key"); + yield* fs.writeFileString(path.join(source, "data", "fixture"), "second"); + yield* sourceStore.saveSnapshot("same-key"); + yield* fs.writeFileString(path.join(source, "data", "fixture"), "mutated-source"); + yield* fs.remove(source, { recursive: true }); + yield* fs.remove(path.join(target, "data"), { recursive: true }); + yield* fs.makeDirectory(path.join(target, "data")); + expect(yield* (yield* store(target, cache)).restoreSnapshot("same-key")).toBe(true); + expect(yield* fs.readFileString(path.join(target, "data", "fixture"))).toBe("second"); - const target = yield* makeDatabase({ - stackId: `snapshot-${runtime}`, - instanceId: "target", - root, - cacheRoot: "/tmp/supabase-stack-artifacts", - runtime, - }); - const targetPassword = Redacted.make(`snapshot-${runtime}-target-password`); - const targetService = yield* makeService(target.definition, { - id: `database:${runtime}:target`, - config: { ...config, databasePassword: targetPassword }, - }); - const targetRoot = path.join(root, "target"); - if (runtime === "native") yield* fs.makeDirectory(path.join(targetRoot, "data")); - const targetSnapshots = yield* makeDatabaseSnapshots({ - instanceRoot: targetRoot, - runtime, - version: config.version, - stackId: `snapshot-${runtime}`, - instanceId: "target", - }); - yield* targetService.storage( - targetSnapshots - .restoreSnapshot({ source: archive }) - .pipe(Effect.mapError(storageError)), - ); - yield* targetService.start; - yield* targetService.ready; - const rows = yield* query( - yield* target.endpoint, - targetPassword, - "SELECT value FROM snapshot_data", - ); - expect(rows).toEqual([{ value: "restored-row" }]); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), - { timeout: 180_000 }, + yield* fs.writeFileString(path.join(target, "data", "fixture"), "destination"); + expect(yield* fs.readFileString(path.join(target, "data", "fixture"))).toBe("destination"); + const secondTarget = yield* makeInstance("second-target"); + yield* fs.remove(path.join(secondTarget, "data"), { recursive: true }); + yield* fs.makeDirectory(path.join(secondTarget, "data")); + expect(yield* (yield* store(secondTarget, cache)).restoreSnapshot("same-key")).toBe(true); + expect(yield* fs.readFileString(path.join(secondTarget, "data", "fixture"))).toBe("second"); + }), + ), ); -} -it.live("rejects nonempty targets, incompatible versions, and unsafe members", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const sourceRoot = yield* prepareRoot(fs, "native", "snapshot-invalid-source-"); - const destinationRoot = yield* fs.makeTempDirectoryScoped({ prefix: "snapshot-invalid-" }); - const source = yield* makeDatabaseSnapshots({ - instanceRoot: sourceRoot, - runtime: "native", - version: "17", - stackId: "stack", - instanceId: "source", - }); - const archive = `${destinationRoot}/baseline.tar`; - yield* source.exportSnapshot({ destination: archive }); + it.live("returns an absent-key miss without changing an empty target", () => + live( + Effect.gen(function* () { + const { fs, path, cache, makeInstance } = yield* setup; + const target = yield* makeInstance("target"); + yield* fs.remove(path.join(target, "data", "PG_VERSION")); + const snapshots = yield* store(target, cache); + expect(yield* snapshots.restoreSnapshot("missing")).toBe(false); + expect(yield* fs.readDirectory(path.join(target, "data"))).toEqual([]); + }), + ), + ); - const nonempty = yield* makeDatabaseSnapshots({ - instanceRoot: yield* prepareRoot(fs, "native", "snapshot-nonempty-"), - runtime: "native", - version: "17", - stackId: "stack", - instanceId: "nonempty", - }); - const nonemptyFailure = yield* nonempty - .restoreSnapshot({ source: archive }) - .pipe(Effect.flip); - expect(nonemptyFailure.operation).toBe("restore"); + it.live("fails a miss on a nonempty target and retains its contents", () => + live( + Effect.gen(function* () { + const { fs, path, cache, makeInstance } = yield* setup; + const target = yield* makeInstance("target"); + yield* fs.writeFileString(path.join(target, "data", "keep"), "safe"); + const failure = yield* (yield* store(target, cache)) + .restoreSnapshot("missing") + .pipe(Effect.flip); + expect(failure.operation).toBe("restore"); + expect(yield* fs.readFileString(path.join(target, "data", "keep"))).toBe("safe"); + }), + ), + ); + + it.live("evicts the oldest entry by touch time and restore updates the LRU time", () => + live( + Effect.gen(function* () { + const { fs, path, cache, makeInstance } = yield* setup; + const source = yield* makeInstance("source"); + const snapshots = yield* store(source, cache); + for (const key of ["one", "two", "three"]) { + yield* fs.writeFileString(path.join(source, "data", "fixture"), key); + yield* snapshots.saveSnapshot(key); + } + const saved = yield* entries(fs, path, cache); + for (const [key, seconds] of [ + ["one", 1_000], + ["two", 2_000], + ["three", 3_000], + ] as const) { + const entry = saved.find((candidate) => candidate.key === key); + if (entry === undefined) throw new Error(`Missing ${key}`); + yield* fs.utimes(entryPath(path, cache, entry.name), seconds, seconds); + } + yield* fs.remove(path.join(source, "data"), { recursive: true }); + yield* fs.makeDirectory(path.join(source, "data")); + const one = saved.find((entry) => entry.key === "one"); + if (one === undefined) throw new Error("Missing one"); + const beforeRestore = yield* Clock.currentTimeMillis; + expect(yield* snapshots.restoreSnapshot("one")).toBe(true); + const afterRestore = yield* Clock.currentTimeMillis; + const touched = Option.match((yield* fs.stat(entryPath(path, cache, one.name))).mtime, { + onNone: () => undefined, + onSome: (mtime) => mtime.getTime(), + }); + if (touched === undefined) throw new Error("Snapshot touch time is unavailable"); + expect(touched).toBeGreaterThanOrEqual(beforeRestore - 1_000); + expect(touched).toBeLessThanOrEqual(afterRestore + 1_000); + yield* fs.writeFileString(path.join(source, "data", "fixture"), "four"); + yield* snapshots.saveSnapshot("four"); + expect((yield* entries(fs, path, cache)).map((entry) => entry.key).sort()).toEqual([ + "four", + "one", + "three", + ]); + }), + ), + ); - const incompatibleRoot = yield* prepareRoot(fs, "native", "snapshot-version-"); - yield* fs.remove(`${incompatibleRoot}/data/PG_VERSION`); - yield* writeMarker(fs, incompatibleRoot, "native", "15.14.1.173"); - const incompatible = yield* makeDatabaseSnapshots({ - instanceRoot: incompatibleRoot, - runtime: "native", - version: "15", - stackId: "stack", - instanceId: "version", - }); - const versionFailure = yield* incompatible - .restoreSnapshot({ source: archive }) - .pipe(Effect.flip); - expect(versionFailure.operation).toBe("descriptor"); + it.live("treats an incompatible descriptor as a full-key miss", () => + live( + Effect.gen(function* () { + const { fs, path, cache, makeInstance } = yield* setup; + const source = yield* makeInstance("source"); + const snapshots = yield* store(source, cache); + yield* snapshots.saveSnapshot("old-version"); + const saved = yield* entries(fs, path, cache); + const entry = saved.find((candidate) => candidate.key === "old-version"); + if (entry === undefined) throw new Error("Missing old-version"); + const descriptorPath = path.join(entryPath(path, cache, entry.name), "descriptor.json"); + const descriptor = yield* fs.readFileString(descriptorPath); + yield* fs.writeFileString(descriptorPath, descriptor.replace(version, "15.14.1.173")); + yield* fs.remove(path.join(source, "data"), { recursive: true }); + yield* fs.makeDirectory(path.join(source, "data")); + expect(yield* snapshots.restoreSnapshot("old-version")).toBe(false); + expect(yield* fs.readDirectory(path.join(source, "data"))).toEqual([]); + }), + ), + ); - const unsafeSource = yield* prepareRoot(fs, "native", "snapshot-unsafe-source-"); - yield* fs.symlink("/tmp", `${unsafeSource}/data/unsafe-link`); - const unsafe = yield* makeDatabaseSnapshots({ - instanceRoot: unsafeSource, - runtime: "native", - version: "17", - stackId: "stack", - instanceId: "unsafe", - }); - const unsafeArchive = `${destinationRoot}/unsafe.tar`; - yield* unsafe.exportSnapshot({ destination: unsafeArchive }); - const unsafeTarget = yield* prepareRoot(fs, "native", "snapshot-unsafe-target-"); - yield* fs.remove(`${unsafeTarget}/data/PG_VERSION`); - const unsafeRestore = yield* makeDatabaseSnapshots({ - instanceRoot: unsafeTarget, - runtime: "native", - version: "17", - stackId: "stack", - instanceId: "unsafe-target", - }); - const unsafeFailure = yield* unsafeRestore - .restoreSnapshot({ source: unsafeArchive }) - .pipe(Effect.flip); - expect(unsafeFailure.operation).toBe("validate"); + it.live("rejects corrupt manifests and data before changing the target", () => + live( + Effect.gen(function* () { + const { fs, path, cache, makeInstance } = yield* setup; + const source = yield* makeInstance("source"); + const snapshots = yield* store(source, cache); + yield* snapshots.saveSnapshot("corrupt-manifest"); + const saved = yield* entries(fs, path, cache); + const manifest = saved.find((candidate) => candidate.key === "corrupt-manifest"); + if (manifest === undefined) throw new Error("Missing corrupt-manifest"); + yield* fs.writeFileString( + path.join(entryPath(path, cache, manifest.name), "descriptor.json"), + "{}", + ); + yield* fs.remove(path.join(source, "data"), { recursive: true }); + yield* fs.makeDirectory(path.join(source, "data")); + const manifestFailure = yield* snapshots + .restoreSnapshot("corrupt-manifest") + .pipe(Effect.flip); + expect(manifestFailure.operation).toBe("descriptor"); + expect(yield* fs.readDirectory(path.join(source, "data"))).toEqual([]); - const descriptor = yield* tar(["-xOf", archive, "metadata/descriptor.json"]); - const malformedRoot = yield* fs.makeTempDirectoryScoped({ prefix: "snapshot-malformed-" }); - yield* fs.makeDirectory(`${malformedRoot}/data`, { recursive: true }); - yield* fs.makeDirectory(`${malformedRoot}/metadata`, { recursive: true }); - yield* fs.writeFileString(`${malformedRoot}/metadata/descriptor.json`, descriptor); - const malformedArchive = `${destinationRoot}/missing-pg-version.tar`; - yield* tar(["-cf", malformedArchive, "-C", malformedRoot, "data", "metadata"]); - const malformedTarget = yield* fs.makeTempDirectoryScoped({ - prefix: "snapshot-malformed-target-", - }); - yield* fs.makeDirectory(`${malformedTarget}/data`, { recursive: true }); - const malformedStore = yield* makeDatabaseSnapshots({ - instanceRoot: malformedTarget, - runtime: "native", - version: "17", - stackId: "stack", - instanceId: "malformed", - }); - const malformedFailure = yield* malformedStore - .restoreSnapshot({ source: malformedArchive }) - .pipe(Effect.flip); - expect(malformedFailure.operation).toBe("validate"); - expect(yield* fs.readDirectory(`${malformedTarget}/data`)).toEqual([]); + yield* fs.writeFileString(path.join(source, "data", "PG_VERSION"), "17\n"); + yield* fs.writeFileString(path.join(source, "data", "fixture"), "invalid"); + yield* snapshots.saveSnapshot("corrupt-data"); + const dataEntry = (yield* entries(fs, path, cache)).find( + (candidate) => candidate.key === "corrupt-data", + ); + if (dataEntry === undefined) throw new Error("Missing corrupt-data"); + yield* fs.writeFileString( + path.join(entryPath(path, cache, dataEntry.name), "data", "PG_VERSION"), + "16\n", + ); + yield* fs.remove(path.join(source, "data"), { recursive: true }); + yield* fs.makeDirectory(path.join(source, "data")); + const dataFailure = yield* snapshots.restoreSnapshot("corrupt-data").pipe(Effect.flip); + expect(dataFailure.operation).toBe("validate"); + expect(yield* fs.readDirectory(path.join(source, "data"))).toEqual([]); + }), + ), + ); - yield* fs.writeFileString(`${malformedRoot}/data/PG_VERSION`, "16\n"); - const wrongVersionArchive = `${destinationRoot}/wrong-pg-version.tar`; - yield* tar(["-cf", wrongVersionArchive, "-C", malformedRoot, "data", "metadata"]); - const wrongVersionTarget = yield* fs.makeTempDirectoryScoped({ - prefix: "snapshot-wrong-version-target-", - }); - yield* fs.makeDirectory(`${wrongVersionTarget}/data`, { recursive: true }); - const wrongVersionStore = yield* makeDatabaseSnapshots({ - instanceRoot: wrongVersionTarget, - runtime: "native", - version: "17", - stackId: "stack", - instanceId: "wrong-version", - }); - const wrongVersionFailure = yield* wrongVersionStore - .restoreSnapshot({ source: wrongVersionArchive }) - .pipe(Effect.flip); - expect(wrongVersionFailure.operation).toBe("validate"); - expect(yield* fs.readDirectory(`${wrongVersionTarget}/data`)).toEqual([]); + it.live("completes concurrent same-key saves with complete generations", () => + live( + Effect.gen(function* () { + const { fs, path, cache, makeInstance } = yield* setup; + const first = yield* makeInstance("first"); + const second = yield* makeInstance("second"); + for (const [root, value] of [ + [first, "first"], + [second, "second"], + ] as const) { + yield* fs.writeFileString(path.join(root, "data", "generation"), value); + yield* fs.makeDirectory(path.join(root, "data", "nested")); + yield* fs.writeFileString(path.join(root, "data", "nested", "generation"), value); + } + const results = yield* Effect.all( + [ + (yield* store(first, cache)).saveSnapshot("same-key"), + (yield* store(second, cache)).saveSnapshot("same-key"), + ], + { concurrency: "unbounded" }, + ); + expect(results).toHaveLength(2); + const target = yield* makeInstance("target"); + yield* fs.remove(path.join(target, "data"), { recursive: true }); + yield* fs.makeDirectory(path.join(target, "data")); + expect(yield* (yield* store(target, cache)).restoreSnapshot("same-key")).toBe(true); + const generation = yield* fs.readFileString(path.join(target, "data", "generation")); + expect(["first", "second"]).toContain(generation); + expect(yield* fs.readFileString(path.join(target, "data", "nested", "generation"))).toBe( + generation, + ); + }), + ), + ); - const dockerRoot = yield* fs.makeTempDirectoryScoped({ - prefix: "snapshot-docker-unsafe-source-", - }); - yield* fs.makeDirectory(`${dockerRoot}/data`, { recursive: true }); - yield* fs.makeDirectory(`${dockerRoot}/metadata`, { recursive: true }); - yield* fs.writeFileString(`${dockerRoot}/data/PG_VERSION`, "17\n"); - yield* fs.symlink("/tmp", `${dockerRoot}/data/unsafe-link`); - const dockerDescriptor = yield* Schema.encodeEffect( - Schema.fromJsonString(SnapshotDescriptor), - )({ - format: "supabase-database-snapshot-v1", - version: "17.6.1.173", - runtime: "docker", - platform: process.platform, - arch: process.arch, - profile: "supabase", - }); - yield* fs.writeFileString(`${dockerRoot}/metadata/descriptor.json`, dockerDescriptor); - const dockerArchive = `${destinationRoot}/unsafe-docker.tar`; - yield* tar(["-cf", dockerArchive, "-C", dockerRoot, "data", "metadata"]); - const dockerTarget = yield* fs.makeTempDirectoryScoped({ - prefix: "snapshot-docker-unsafe-target-", - }); - yield* fs.makeDirectory(`${dockerTarget}/data`, { recursive: true }); - const dockerStore = yield* makeDatabaseSnapshots({ - instanceRoot: dockerTarget, - runtime: "docker", - version: "17", - stackId: "stack", - instanceId: "docker-unsafe", - }); - const dockerFailure = yield* dockerStore - .restoreSnapshot({ source: dockerArchive }) - .pipe(Effect.flip); - expect(dockerFailure.operation).toBe("validate"); + it.live("reclaims stale stages before the next operation", () => + live( + Effect.gen(function* () { + const { fs, path, cache, makeInstance } = yield* setup; + const source = yield* makeInstance("source"); + const stageRoot = path.join(cache, "stack-database-snapshots", "stages"); + yield* fs.makeDirectory(path.join(stageRoot, "abandoned"), { recursive: true }); + const staleRestore = path.join(source, ".supabase-restore-abandoned", "data"); + yield* fs.makeDirectory(staleRestore, { recursive: true }); + yield* (yield* store(source, cache)).saveSnapshot("cleanup"); + expect(yield* fs.exists(path.join(stageRoot, "abandoned"))).toBe(false); + expect(yield* fs.exists(path.join(source, ".supabase-restore-abandoned"))).toBe(false); + }), + ), + ); - const dockerNonemptyRoot = yield* fs.makeTempDirectoryScoped({ - prefix: "snapshot-docker-nonempty-target-", - }); - yield* fs.makeDirectory(`${dockerNonemptyRoot}/data`, { recursive: true }); - yield* fs.writeFileString(`${dockerNonemptyRoot}/data/sentinel`, "occupied\n"); - const dockerNonemptyStore = yield* makeDatabaseSnapshots({ - instanceRoot: dockerNonemptyRoot, - runtime: "docker", - version: "17", - stackId: "stack", - instanceId: "docker-nonempty", - }); - const dockerNonemptyFailure = yield* dockerNonemptyStore - .restoreSnapshot({ source: archive }) - .pipe(Effect.flip); - expect(dockerNonemptyFailure.operation).toBe("restore"); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), -); + it.live("rolls back a failed marker publication", () => + live( + Effect.gen(function* () { + const { fs, path, cache, makeInstance } = yield* setup; + const source = yield* makeInstance("source"); + const target = yield* makeInstance("target"); + yield* fs.writeFileString(path.join(source, "data", "fixture"), "snapshot"); + yield* (yield* store(source, cache)).saveSnapshot("rollback"); + yield* fs.remove(path.join(target, "data"), { recursive: true }); + yield* fs.makeDirectory(path.join(target, "data")); + let fail = true; + const failingFs: FileSystem.FileSystem = { + ...fs, + rename: (from, to) => { + if ( + fail && + from.endsWith("/ready.json") && + to.endsWith(".supabase-database-ready.json") + ) { + fail = false; + return Effect.die(new Error("injected marker publication failure")); + } + return fs.rename(from, to); + }, + }; + const result = yield* (yield* store(target, cache).pipe( + Effect.provideService(FileSystem.FileSystem, failingFs), + )) + .restoreSnapshot("rollback") + .pipe(Effect.exit); + expect(Exit.isFailure(result)).toBe(true); + expect( + yield* fs.readFileString(path.join(target, ".supabase-database-ready.json")), + ).toContain(version); + expect(yield* fs.readDirectory(path.join(target, "data"))).toEqual([]); + }), + ), + ); +}); diff --git a/packages/stack/src/services/DatabaseSnapshot.ts b/packages/stack/src/services/DatabaseSnapshot.ts index 6f512a54c7..594ea74c8c 100644 --- a/packages/stack/src/services/DatabaseSnapshot.ts +++ b/packages/stack/src/services/DatabaseSnapshot.ts @@ -1,18 +1,32 @@ -import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { Crypto, Data, Effect, FileSystem, Path, Schema, Stream } from "effect"; -import { postgresVersion, resolveArtifact } from "../Artifacts.ts"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- SQLite is the cross-process lock primitive. +import { DatabaseSync } from "node:sqlite"; +import { + Clock, + Crypto, + Data, + Effect, + Exit, + FileSystem, + Option, + Path, + Predicate, + Schedule, + Schema, +} from "effect"; +import { postgresVersion } from "../Artifacts.ts"; +import { copyDirectory } from "../storage/DirectoryCopy.ts"; import type { DatabaseRuntime } from "./Database.ts"; -import { makeContainerRuntime, type ContainerRuntime } from "../runtime/Container.ts"; -export const SnapshotDescriptor = Schema.Struct({ +const SnapshotDescriptor = Schema.Struct({ format: Schema.Literal("supabase-database-snapshot-v1"), version: Schema.String, runtime: Schema.Literals(["native", "docker", "podman"]), platform: Schema.String, arch: Schema.String, profile: Schema.Literal("supabase"), + logicalKey: Schema.String, + keyDigest: Schema.String, }); -export interface SnapshotDescriptor extends Schema.Schema.Type {} export class DatabaseSnapshotError extends Data.TaggedError("DatabaseSnapshotError")<{ readonly operation: string; @@ -20,17 +34,6 @@ export class DatabaseSnapshotError extends Data.TaggedError("DatabaseSnapshotErr readonly cause?: unknown; }> {} -export interface DatabaseSnapshot { - readonly descriptor: SnapshotDescriptor; - readonly destination: string; -} - -const ReadyMarker = Schema.Struct({ - version: Schema.String, - runtime: Schema.Literals(["native", "docker", "podman"]), - profile: Schema.Literal("supabase"), -}); - const errorFor = (operation: string, cause: unknown) => cause instanceof DatabaseSnapshotError ? cause @@ -40,14 +43,22 @@ const errorFor = (operation: string, cause: unknown) => cause, }); -const safeRelative = (name: string) => { - if (name.startsWith("/") || name.includes("\\")) return false; - const parts = name.split("/"); - return parts.every((part) => part !== "" && part !== "." && part !== ".."); -}; +const ReadyMarker = Schema.Struct({ + version: Schema.String, + runtime: Schema.Literals(["native", "docker", "podman"]), + profile: Schema.Literal("supabase"), +}); + +const markerText = (version: string, runtime: DatabaseRuntime) => + Schema.encodeEffect(Schema.fromJsonString(ReadyMarker))({ + version, + runtime, + profile: "supabase", + }); export const makeDatabaseSnapshots = Effect.fn("DatabaseSnapshot.make")(function* (options: { readonly instanceRoot: string; + readonly cacheRoot: string; readonly runtime: DatabaseRuntime; readonly version: string; readonly stackId: string; @@ -56,499 +67,319 @@ export const makeDatabaseSnapshots = Effect.fn("DatabaseSnapshot.make")(function const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const version = postgresVersion(options.version); - const container: ContainerRuntime | undefined = - options.runtime === "native" - ? undefined - : yield* makeContainerRuntime({ engine: options.runtime }); - const helperImage = - options.runtime === "native" - ? undefined - : yield* resolveArtifact({ service: "database", version }).pipe( - Effect.mapError((cause) => errorFor("artifact", cause)), - ); - const descriptor: SnapshotDescriptor = { - format: "supabase-database-snapshot-v1", + const root = path.join(options.cacheRoot, "stack-database-snapshots"); + const entries = path.join(root, "entries"); + const stages = path.join(root, "stages"); + const lockFile = path.join(root, ".lock.sqlite"); + const base = { + format: "supabase-database-snapshot-v1" as const, version, runtime: options.runtime, platform: process.platform, arch: process.arch, - profile: "supabase", + profile: "supabase" as const, }; + const mapError = (operation: string, effect: Effect.Effect) => + effect.pipe(Effect.mapError((cause) => errorFor(operation, cause))); + const ensureStore = mapError( + "storage", + fs + .makeDirectory(root, { recursive: true, mode: 0o700 }) + .pipe(Effect.andThen(fs.makeDirectory(entries, { recursive: true, mode: 0o700 }))), + ); - const commandRaw = Effect.fn("DatabaseSnapshot.command")(function* (args: ReadonlyArray) { - return yield* Effect.scoped( - Effect.gen(function* () { - const child = yield* spawner.spawn(ChildProcess.make("tar", args, { stdin: "ignore" })); - const [stdout, stderr, code] = yield* Effect.all( - [ - child.stdout.pipe( - Stream.decodeText, - Stream.runFold( - () => "", - (all, chunk) => `${all}${chunk}`.slice(0, 65536), - ), - ), - child.stderr.pipe( - Stream.decodeText, - Stream.runFold( - () => "", - (all, chunk) => `${all}${chunk}`.slice(0, 65536), - ), - ), - child.exitCode, - ], - { concurrency: "unbounded" }, - ); - if (Number(code) !== 0) - return yield* errorFor("tar", stderr.trim() || `tar exited with ${String(code)}`); - return stdout; + const withLock = (effect: Effect.Effect) => + Effect.acquireUseRelease( + Effect.try({ + try: () => new DatabaseSync(lockFile), + catch: (cause) => errorFor("lock", cause), }), - ); - }); - const command = (args: ReadonlyArray) => - commandRaw(args).pipe(Effect.mapError((cause) => errorFor("tar", cause))); - - const commandLines = Effect.fn("DatabaseSnapshot.commandLines")( - ( - args: ReadonlyArray, - onLine: (line: string) => Effect.Effect, - ) => - Effect.scoped( + (connection) => Effect.gen(function* () { - const child = yield* spawner.spawn(ChildProcess.make("tar", args, { stdin: "ignore" })); - const [, stderr, code] = yield* Effect.all( - [ - child.stdout.pipe(Stream.decodeText, Stream.splitLines, Stream.runForEach(onLine)), - child.stderr.pipe( - Stream.decodeText, - Stream.runFold( - () => "", - (all, chunk) => `${all}${chunk}`.slice(0, 65536), - ), + yield* Effect.try({ + try: () => connection.exec("PRAGMA busy_timeout = 0"), + catch: (cause) => errorFor("lock", cause), + }); + yield* Effect.try({ + try: () => connection.exec("BEGIN IMMEDIATE"), + catch: (cause) => errorFor("lock", cause), + }).pipe( + Effect.retry({ + schedule: Schedule.spaced("50 millis").pipe( + Schedule.upTo({ duration: "120 seconds" }), ), - child.exitCode, - ], - { concurrency: "unbounded" }, + while: (cause) => + Predicate.hasProperty(cause.cause, "errcode") && cause.cause.errcode === 5, + }), + Effect.mapError((cause) => errorFor("lock", cause)), ); - if (Number(code) !== 0) - return yield* errorFor("tar", stderr.trim() || `tar exited with ${String(code)}`); + return yield* effect; + }), + (connection) => + Effect.try({ + try: () => connection.close(), + catch: (cause) => errorFor("unlock", cause), + }), + ); + + const keyDescriptor = (key: string) => ({ ...base, logicalKey: key }); + const digest = (key: string) => + Schema.encodeEffect( + Schema.fromJsonString( + Schema.Struct({ + format: Schema.Literal("supabase-database-snapshot-v1"), + version: Schema.String, + runtime: Schema.Literals(["native", "docker", "podman"]), + platform: Schema.String, + arch: Schema.String, + profile: Schema.Literal("supabase"), + logicalKey: Schema.String, }), - ).pipe( - Effect.mapError((cause) => - cause instanceof DatabaseSnapshotError ? cause : errorFor("tar", cause), - ), ), - ); + )(keyDescriptor(key)).pipe( + Effect.flatMap((encoded) => crypto.digest("SHA-256", new TextEncoder().encode(encoded))), + Effect.map((bytes) => + Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""), + ), + Effect.mapError((cause) => errorFor("key", cause)), + ); - const toolCommand = Effect.fn("DatabaseSnapshot.toolCommand")( - ( - stage: string, - script: string, - writableInstance = false, - onLine?: (line: string) => Effect.Effect, - ) => - Effect.gen(function* () { - if (container === undefined) - return yield* errorFor("container", "Container runtime missing"); - if (helperImage === undefined) - return yield* errorFor("container", "Container runtime missing"); - yield* container - .prepare(helperImage.image) - .pipe(Effect.mapError((cause) => errorFor("container", cause))); - return yield* Effect.scoped( - Effect.gen(function* () { - const acquire = container - .launchTool({ - image: helperImage.image, - stackId: options.stackId, - instanceId: options.instanceId, - entrypoint: "/usr/bin/busybox", - env: {}, - mounts: [ - { - source: options.instanceRoot, - target: "/instance", - readOnly: !writableInstance, - }, - { source: stage, target: "/stage", readOnly: false }, - ], - args: ["sh", "-c", script], - }) - .pipe( - Effect.catchTag("ContainerLaunchError", ({ failure, process: partial }) => - partial.stop.pipe( - Effect.andThen(partial.remove), - Effect.mapError((cause) => errorFor("container-cleanup", cause)), - Effect.andThen(Effect.fail(errorFor("container", failure))), - ), - ), - ); - return yield* Effect.acquireUseRelease( - acquire, - (owned) => - Effect.all( - [ - onLine === undefined - ? owned.stdout.pipe( - Stream.decodeText, - Stream.runFold( - () => "", - (all, chunk) => `${all}${chunk}`.slice(0, 65536), - ), - ) - : owned.stdout.pipe( - Stream.decodeText, - Stream.splitLines, - Stream.runForEach(onLine), - Effect.as(""), - ), - owned.stderr.pipe( - Stream.decodeText, - Stream.runFold( - () => "", - (all, chunk) => `${all}${chunk}`.slice(0, 65536), - ), - ), - owned.exitCode, - ], - { concurrency: "unbounded" }, - ).pipe( - Effect.flatMap(([stdout, stderr, code]) => - Number(code) === 0 - ? Effect.succeed(stdout) - : Effect.fail( - errorFor( - "container", - stderr.trim() || `Helper exited with ${String(code)}`, - ), - ), - ), - ), - (owned) => - owned.stop.pipe( - Effect.andThen(owned.remove), - Effect.mapError((cause) => errorFor("container", cause)), - ), - ); - }), - ); - }).pipe(Effect.mapError((cause) => errorFor("container", cause))), + const copy = (source: string, destination: string) => + copyDirectory(source, destination).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.mapError((cause) => errorFor("copy", cause)), + ); + + const readReady = mapError( + "ready", + fs + .readFileString(path.join(options.instanceRoot, ".supabase-database-ready.json")) + .pipe(Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(ReadyMarker)))), ); - const cleanupStage = Effect.fn("DatabaseSnapshot.cleanupStage")((directory: string) => + const sourceData = Effect.gen(function* () { + const ready = yield* readReady; + if (ready.version !== version || ready.runtime !== options.runtime) + return yield* errorFor("ready", "Database readiness marker does not match the instance"); + const data = path.join(options.instanceRoot, "data"); + if (!(yield* mapError("data", fs.exists(data)))) + return yield* errorFor("data", "Data is absent"); + if (yield* mapError("data", fs.exists(path.join(data, "postmaster.pid")))) + return yield* errorFor("data", "Database must be stopped before snapshotting"); + const pgVersion = yield* mapError("data", fs.readFileString(path.join(data, "PG_VERSION"))); + if (pgVersion.trim() !== version.split(".")[0]) + return yield* errorFor("data", "Database PostgreSQL major version is incompatible"); + return data; + }); + + const cleanupChildren = (directory: string, predicate: (name: string) => boolean) => Effect.gen(function* () { - if ( - options.runtime !== "native" && - ((yield* fs.exists(path.join(directory, "input"))) || - (yield* fs.exists(path.join(directory, "extracted")))) - ) { - yield* toolCommand(directory, "/usr/bin/busybox rm -rf /stage/extracted /stage/input"); + const names = yield* mapError("cleanup", fs.readDirectory(directory)); + for (const name of names) { + if (predicate(name)) + yield* mapError("cleanup", fs.remove(path.join(directory, name), { recursive: true })); } - yield* fs.remove(directory, { recursive: true, force: true }); - }).pipe(Effect.mapError((cause) => errorFor("cleanup", cause))), - ); + }); - const readReady = Effect.fn("DatabaseSnapshot.readReady")(() => + const cleanupStaleStages = Effect.gen(function* () { + yield* cleanupChildren(stages, (name) => name !== "." && name !== ".."); + yield* cleanupChildren(options.instanceRoot, (name) => name.startsWith(".supabase-restore-")); + }); + + const touch = (target: string) => Effect.gen(function* () { - const marker = yield* fs.readFileString( - path.join(options.instanceRoot, ".supabase-database-ready.json"), - ); - return yield* Schema.decodeEffect(Schema.fromJsonString(ReadyMarker))(marker); - }).pipe(Effect.mapError((cause) => errorFor("ready", cause))), - ); + const now = yield* Clock.currentTimeMillis; + yield* mapError("touch", fs.utimes(target, now / 1_000, now / 1_000)); + }); - const verifyData = Effect.fn("DatabaseSnapshot.verifyData")(function* () { - const data = path.join(options.instanceRoot, "data"); - if (options.runtime === "native") { - if (!(yield* fs.exists(data).pipe(Effect.mapError((cause) => errorFor("data", cause))))) - return yield* errorFor("data", "Database data directory is absent"); - if ( - yield* fs - .exists(path.join(data, "postmaster.pid")) - .pipe(Effect.mapError((cause) => errorFor("data", cause))) - ) - return yield* errorFor("data", "Database must be stopped before snapshotting"); - if ( - !(yield* fs - .exists(path.join(data, "PG_VERSION")) - .pipe(Effect.mapError((cause) => errorFor("data", cause)))) - ) - return yield* errorFor("data", "Database data directory is not initialized"); - } else { - yield* toolCommand( - options.instanceRoot, - "test -d /instance/data && test -f /instance/data/PG_VERSION && test ! -e /instance/data/postmaster.pid", - ).pipe( - Effect.mapError((cause) => - cause instanceof DatabaseSnapshotError && cause.operation === "container" - ? new DatabaseSnapshotError({ - operation: "data", - message: cause.message, - cause, - }) - : cause, - ), - ); - } - return data; - }); + const retention = (current: string) => + Effect.gen(function* () { + const names = yield* mapError("retention", fs.readDirectory(entries)); + const values: Array<{ readonly name: string; readonly mtime: number }> = []; + for (const name of names) { + if (name === current) continue; + const info = yield* mapError("retention", fs.stat(path.join(entries, name))); + values.push({ + name, + mtime: Option.match(info.mtime, { + onNone: () => 0, + onSome: (mtime) => mtime.getTime(), + }), + }); + } + values.sort((left, right) => left.mtime - right.mtime || left.name.localeCompare(right.name)); + for (const value of values.slice(0, Math.max(0, values.length - 2))) + yield* mapError( + "retention", + fs.remove(path.join(entries, value.name), { recursive: true }), + ); + }); - const exportSnapshot = Effect.fn("DatabaseSnapshot.export")(function* ({ - destination, - }: { - readonly destination: string; - }) { - const ready = yield* readReady(); - if ( - ready.version !== version || - ready.runtime !== options.runtime || - ready.profile !== "supabase" - ) - return yield* errorFor("ready", "Database readiness marker does not match the instance"); - yield* verifyData(); - if (yield* fs.exists(destination).pipe(Effect.mapError((cause) => errorFor("export", cause)))) - return yield* errorFor("export", "Destination already exists"); - const parent = path.dirname(destination); - const stageToken = yield* crypto.randomUUIDv4.pipe( - Effect.mapError((cause) => errorFor("export", cause)), + const saveSnapshot = Effect.fn("DatabaseSnapshot.save")(function* (logicalKey: string) { + const source = yield* sourceData; + const keyDigest = yield* digest(logicalKey); + const descriptor = { ...keyDescriptor(logicalKey), keyDigest }; + const encoded = yield* mapError( + "descriptor", + Schema.encodeEffect(Schema.fromJsonString(SnapshotDescriptor))(descriptor), ); - return yield* Effect.acquireUseRelease( - fs - .makeTempDirectory({ directory: parent, prefix: `.supabase-snapshot-${stageToken}-` }) - .pipe(Effect.mapError((cause) => errorFor("export", cause))), - (stage) => - Effect.gen(function* () { - const metadata = path.join(stage, "metadata"); - yield* fs - .makeDirectory(metadata, { recursive: true }) - .pipe(Effect.mapError((cause) => errorFor("stage", cause))); - const encoded = yield* Schema.encodeEffect(Schema.fromJsonString(SnapshotDescriptor))( - descriptor, - ).pipe(Effect.mapError((cause) => errorFor("descriptor", cause))); - yield* fs - .writeFileString(path.join(metadata, "descriptor.json"), encoded, { mode: 0o600 }) - .pipe(Effect.mapError((cause) => errorFor("descriptor", cause))); - const archive = path.join(stage, "snapshot.tar"); - if (options.runtime === "native") - yield* command([ - "-cf", - archive, - "-C", - options.instanceRoot, - "data", - "-C", - stage, - "metadata", - ]); - else { - yield* fs - .writeFileString(archive, "", { mode: 0o600 }) - .pipe(Effect.mapError((cause) => errorFor("stage", cause))); - yield* toolCommand( - stage, - "/usr/bin/busybox mkdir -p /stage/input && /usr/bin/busybox cp -R /instance/data /stage/input/data && /usr/bin/busybox cp -R /stage/metadata /stage/input/metadata && /usr/bin/busybox tar -cf /stage/snapshot.tar -C /stage/input data metadata", - ); - } - yield* fs - .link(archive, destination) - .pipe(Effect.mapError((cause) => errorFor("publish", cause))); - yield* fs.remove(archive).pipe(Effect.mapError((cause) => errorFor("publish", cause))); - return { descriptor, destination }; - }), - cleanupStage, + const target = path.join(entries, keyDigest); + yield* ensureStore; + yield* withLock( + Effect.gen(function* () { + yield* mapError("storage", fs.makeDirectory(stages, { recursive: true, mode: 0o700 })); + yield* cleanupStaleStages; + const token = yield* crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => errorFor("stage", cause)), + ); + const stage = path.join(stages, token); + const retired = path.join(stages, "retired-" + token); + yield* mapError("stage", fs.makeDirectory(stage, { recursive: true, mode: 0o700 })); + yield* Effect.acquireUseRelease( + Effect.succeed(stage), + () => + Effect.gen(function* () { + yield* copy(source, path.join(stage, "data")); + yield* mapError( + "descriptor", + fs.writeFileString(path.join(stage, "descriptor.json"), encoded, { mode: 0o600 }), + ); + const hadTarget = yield* mapError("publish", fs.exists(target)); + if (hadTarget) yield* mapError("publish", fs.rename(target, retired)); + const publication = yield* Effect.exit(mapError("publish", fs.rename(stage, target))); + if (Exit.isFailure(publication)) { + if (hadTarget) yield* mapError("rollback", fs.rename(retired, target)); + return yield* Effect.failCause(publication.cause); + } + yield* mapError("publish", fs.remove(retired, { recursive: true, force: true })); + yield* touch(target); + yield* retention(keyDigest); + }), + () => mapError("cleanup", fs.remove(stage, { recursive: true, force: true })), + ); + }), ); }); - const restoreSnapshot = Effect.fn("DatabaseSnapshot.restore")(function* ({ - source, - }: { - readonly source: string; - }) { + const restoreSnapshot = Effect.fn("DatabaseSnapshot.restore")(function* (logicalKey: string) { const data = path.join(options.instanceRoot, "data"); - const dataExists = yield* fs - .exists(data) - .pipe(Effect.mapError((cause) => errorFor("restore", cause))); - if (options.runtime === "native") { - if ( - dataExists && - (yield* fs.readDirectory(data).pipe(Effect.mapError((cause) => errorFor("restore", cause)))) - .length > 0 - ) - return yield* errorFor("restore", "Restore target data directory must be empty"); - } else { - yield* toolCommand( - options.instanceRoot, - 'test ! -e /instance/data || test -z "$(/usr/bin/busybox ls -A /instance/data)"', - true, - ).pipe( - Effect.mapError((cause) => - cause instanceof DatabaseSnapshotError && cause.operation === "container" - ? new DatabaseSnapshotError({ - operation: "restore", - message: cause.message, - cause, - }) - : cause, - ), - ); - } - const stageToken = yield* crypto.randomUUIDv4.pipe( - Effect.mapError((cause) => errorFor("restore", cause)), - ); - return yield* Effect.acquireUseRelease( - fs - .makeTempDirectory({ - directory: options.instanceRoot, - prefix: `.supabase-restore-${stageToken}-`, - }) - .pipe(Effect.mapError((cause) => errorFor("restore", cause))), - (stage) => - Effect.gen(function* () { - const archive = path.join(stage, "source.tar"); - yield* fs - .copyFile(source, archive) - .pipe(Effect.mapError((cause) => errorFor("stage", cause))); - const descriptorText = - options.runtime === "native" - ? yield* command(["-xOf", archive, "metadata/descriptor.json"]) - : yield* toolCommand( - stage, - "/usr/bin/busybox tar -xOf /stage/source.tar metadata/descriptor.json", - ); - const incoming = yield* Schema.decodeEffect(Schema.fromJsonString(SnapshotDescriptor))( - descriptorText, - ).pipe(Effect.mapError((cause) => errorFor("descriptor", cause))); - if ( - incoming.format !== descriptor.format || - incoming.version !== descriptor.version || - incoming.runtime !== descriptor.runtime || - incoming.platform !== descriptor.platform || - incoming.arch !== descriptor.arch || - incoming.profile !== descriptor.profile - ) - return yield* errorFor( - "descriptor", - "Snapshot is incompatible with this database instance", - ); - const validateName = (line: string) => { - const name = line.replace(/\/$/u, ""); - return name.length === 0 || - ((["data", "metadata", "metadata/descriptor.json"].includes(name) || - name.startsWith("data/")) && - safeRelative(name)) - ? Effect.void - : Effect.fail(errorFor("validate", `Unsafe snapshot member: ${name}`)); - }; - if (options.runtime === "native") yield* commandLines(["-tf", archive], validateName); - else - yield* toolCommand( - stage, - "/usr/bin/busybox tar -tf /stage/source.tar", - false, - validateName, - ); - const validateType = (line: string) => - line.length === 0 || line[0] === "-" || line[0] === "d" - ? Effect.void - : Effect.fail(errorFor("validate", "Snapshot contains a non-regular member")); - if (options.runtime === "native") yield* commandLines(["-tvf", archive], validateType); - else - yield* toolCommand( - stage, - "/usr/bin/busybox tar -tvf /stage/source.tar", - false, - validateType, - ); - const extracted = path.join(stage, "extracted"); - yield* fs - .makeDirectory(extracted, { recursive: true }) - .pipe(Effect.mapError((cause) => errorFor("extract", cause))); - if (options.runtime === "native") yield* command(["-xf", archive, "-C", extracted]); - else - yield* toolCommand( - stage, - "/usr/bin/busybox tar -xf /stage/source.tar -C /stage/extracted", - ); - const expectedMajor = version.split(".")[0]; - if (options.runtime === "native") { - const restoredVersion = path.join(extracted, "data", "PG_VERSION"); - if ( - !(yield* fs - .exists(restoredVersion) - .pipe(Effect.mapError((cause) => errorFor("validate", cause)))) - ) - return yield* errorFor("validate", "Snapshot data is not initialized"); - const actualVersion = yield* fs - .readFileString(restoredVersion) - .pipe(Effect.mapError((cause) => errorFor("validate", cause))); - if (actualVersion.trim() !== expectedMajor) - return yield* errorFor( + const keyDigest = yield* digest(logicalKey); + const target = path.join(entries, keyDigest); + yield* ensureStore; + return yield* withLock( + Effect.gen(function* () { + yield* mapError("storage", fs.makeDirectory(stages, { recursive: true, mode: 0o700 })); + yield* cleanupStaleStages; + if (yield* mapError("restore", fs.exists(data))) { + if ((yield* mapError("restore", fs.readDirectory(data))).length > 0) + return yield* errorFor("restore", "Restore target data directory must be empty"); + } + if (!(yield* mapError("restore", fs.exists(target)))) return false; + + const descriptor = yield* mapError( + "descriptor", + fs + .readFileString(path.join(target, "descriptor.json")) + .pipe(Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(SnapshotDescriptor)))), + ); + if (descriptor.keyDigest !== keyDigest || descriptor.logicalKey !== logicalKey) + return yield* errorFor("descriptor", "Snapshot manifest does not match its key"); + if ( + descriptor.version !== version || + descriptor.runtime !== options.runtime || + descriptor.platform !== process.platform || + descriptor.arch !== process.arch + ) + return false; + + const token = yield* crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => errorFor("stage", cause)), + ); + const stage = path.join(options.instanceRoot, ".supabase-restore-" + token); + const retired = path.join(options.instanceRoot, ".supabase-restore-retired-" + token); + const marker = path.join(options.instanceRoot, ".supabase-database-ready.json"); + const retiredMarker = path.join( + options.instanceRoot, + ".supabase-restore-retired-marker-" + token, + ); + const stagedMarker = path.join(stage, "ready.json"); + yield* mapError("stage", fs.makeDirectory(stage, { recursive: true, mode: 0o700 })); + yield* Effect.acquireUseRelease( + Effect.succeed(stage), + () => + Effect.gen(function* () { + yield* copy(path.join(target, "data"), path.join(stage, "data")); + const stagedData = path.join(stage, "data"); + const stagedVersion = yield* mapError( "validate", - "Snapshot data has an incompatible PostgreSQL major version", + fs.readFileString(path.join(stagedData, "PG_VERSION")), + ); + if (stagedVersion.trim() !== version.split(".")[0]) + return yield* errorFor( + "validate", + "Snapshot PostgreSQL major version is incompatible", + ); + if (yield* mapError("validate", fs.exists(path.join(stagedData, "postmaster.pid")))) + return yield* errorFor("validate", "Snapshot contains a running database"); + const markerContents = yield* markerText(version, options.runtime).pipe( + Effect.mapError((cause) => errorFor("ready", cause)), + ); + yield* mapError( + "ready", + fs.writeFileString(stagedMarker, markerContents, { mode: 0o600 }), ); - } else { - yield* toolCommand( - stage, - "/usr/bin/busybox test -f /stage/extracted/data/PG_VERSION && /usr/bin/busybox cat /stage/extracted/data/PG_VERSION || /usr/bin/busybox echo __missing_pg_version__", - false, - (line) => - line.trim() === expectedMajor - ? Effect.void - : Effect.fail( - errorFor( - "validate", - "Snapshot data is missing or has an incompatible PostgreSQL major version", - ), - ), - ); - } - if (options.runtime === "native") { - yield* fs - .rename(path.join(extracted, "data"), data) - .pipe(Effect.mapError((cause) => errorFor("publish", cause))); - } else { - const stageName = path.basename(stage); - yield* toolCommand( - stage, - `if test -e /instance/data; then /usr/bin/busybox rmdir /instance/data; fi && /usr/bin/busybox mv /instance/${stageName}/extracted/data /instance/data && /usr/bin/busybox chown -R 100:101 /instance/data && /usr/bin/busybox chmod 700 /instance/data`, - true, - ); - } - const marker = yield* Schema.encodeEffect(Schema.fromJsonString(ReadyMarker))({ - version, - runtime: options.runtime, - profile: "supabase", - }).pipe(Effect.mapError((cause) => errorFor("ready", cause))); - yield* fs - .writeFileString( - path.join(options.instanceRoot, ".supabase-database-ready.json"), - marker, - { mode: 0o600 }, - ) - .pipe( - Effect.mapError((cause) => errorFor("ready", cause)), - Effect.catch((failure: DatabaseSnapshotError) => { - const cleanup = - options.runtime === "native" - ? fs - .remove(data, { recursive: true, force: true }) - .pipe(Effect.mapError((cause) => errorFor("cleanup", cause))) - : toolCommand( - options.instanceRoot, - "/usr/bin/busybox rm -rf /instance/data", - true, - ).pipe(Effect.asVoid); - return cleanup.pipe(Effect.andThen(Effect.fail(failure))); - }), - ); - return { descriptor: incoming, destination: source }; - }), - cleanupStage, + // The handoff is short and uninterruptible. If marker publication fails, the old + // complete data tree and marker are restored before the error escapes. + yield* Effect.uninterruptible( + Effect.gen(function* () { + const hadData = yield* mapError("publish", fs.exists(data)); + const hadMarker = yield* mapError("publish", fs.exists(marker)); + if (hadData) yield* mapError("publish", fs.rename(data, retired)); + const dataPublication = yield* Effect.exit( + mapError("publish", fs.rename(path.join(stage, "data"), data)), + ); + if (Exit.isFailure(dataPublication)) { + if (hadData) yield* mapError("rollback", fs.rename(retired, data)); + return yield* Effect.failCause(dataPublication.cause); + } + if (hadMarker) { + const markerRetirement = yield* Effect.exit( + mapError("publish", fs.rename(marker, retiredMarker)), + ); + if (Exit.isFailure(markerRetirement)) { + yield* mapError( + "rollback", + fs.remove(data, { recursive: true, force: true }), + ); + if (hadData) yield* mapError("rollback", fs.rename(retired, data)); + return yield* Effect.failCause(markerRetirement.cause); + } + } + const markerPublication = yield* Effect.exit( + mapError("ready", fs.rename(stagedMarker, marker)), + ); + if (Exit.isFailure(markerPublication)) { + if (hadMarker) yield* mapError("rollback", fs.rename(retiredMarker, marker)); + yield* mapError("rollback", fs.remove(data, { recursive: true, force: true })); + if (hadData) yield* mapError("rollback", fs.rename(retired, data)); + return yield* Effect.failCause(markerPublication.cause); + } + yield* mapError("publish", fs.remove(retired, { recursive: true, force: true })); + if (hadMarker) + yield* mapError("publish", fs.remove(retiredMarker, { force: true })); + }), + ); + yield* touch(target); + }), + () => mapError("cleanup", fs.remove(stage, { recursive: true, force: true })), + ); + return true; + }), ); }); - return { exportSnapshot, restoreSnapshot }; + return { saveSnapshot, restoreSnapshot }; }); diff --git a/packages/stack/src/storage/DirectoryCopy.integration.test.ts b/packages/stack/src/storage/DirectoryCopy.integration.test.ts new file mode 100644 index 0000000000..1fdffc3178 --- /dev/null +++ b/packages/stack/src/storage/DirectoryCopy.integration.test.ts @@ -0,0 +1,124 @@ +import { NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit, FileSystem, Path, Scope } from "effect"; +import { copyDirectory } from "./DirectoryCopy.ts"; + +const run = (effect: Effect.Effect) => + Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); + +describe("copyDirectory", () => { + it.live("copies nested files and preserves modes", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "directory-copy-" }); + const source = path.join(root, "source"); + const destination = path.join(root, "destination"); + const nested = path.join(source, "nested"); + yield* fs.makeDirectory(nested, { recursive: true }); + yield* fs.writeFileString(path.join(source, "root.txt"), "root\n"); + yield* fs.writeFileString(path.join(nested, "child.txt"), "child\n"); + if (process.platform !== "win32") { + yield* fs.chmod(path.join(source, "root.txt"), 0o640); + yield* fs.chmod(nested, 0o750); + } + + yield* copyDirectory(source, destination); + + expect(yield* fs.readFileString(path.join(destination, "root.txt"))).toBe("root\n"); + expect(yield* fs.readFileString(path.join(destination, "nested", "child.txt"))).toBe( + "child\n", + ); + yield* fs.writeFileString(path.join(destination, "root.txt"), "destination\n"); + yield* fs.writeFileString(path.join(source, "nested", "child.txt"), "source\n"); + expect(yield* fs.readFileString(path.join(source, "root.txt"))).toBe("root\n"); + expect(yield* fs.readFileString(path.join(destination, "nested", "child.txt"))).toBe( + "child\n", + ); + if (process.platform !== "win32") { + expect(Number((yield* fs.stat(path.join(destination, "root.txt"))).mode) & 0o777).toBe( + 0o640, + ); + expect(Number((yield* fs.stat(path.join(destination, "nested"))).mode) & 0o777).toBe( + 0o750, + ); + } + }), + ), + ); + + it.live("rejects symlinks and leaves the destination absent", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "directory-copy-link-" }); + const source = path.join(root, "source"); + const destination = path.join(root, "destination"); + yield* fs.makeDirectory(source); + yield* fs.writeFileString(path.join(root, "outside.txt"), "outside\n"); + const symlink = yield* fs + .symlink(path.join(root, "outside.txt"), path.join(source, "link.txt")) + .pipe(Effect.exit); + if (Exit.isFailure(symlink)) { + if ( + process.platform === "win32" && + /(?:EPERM|EACCES|privilege)/iu.test(String(symlink.cause)) + ) + return; + return yield* Effect.failCause(symlink.cause); + } + + const result = yield* copyDirectory(source, destination).pipe(Effect.exit); + + expect(Exit.isFailure(result)).toBe(true); + expect(yield* fs.exists(destination)).toBe(false); + }), + ), + ); + + it.live("rejects an existing destination", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "directory-copy-existing-" }); + const source = `${root}/source`; + const destination = `${root}/destination`; + yield* fs.makeDirectory(source); + yield* fs.makeDirectory(destination); + const result = yield* copyDirectory(source, destination).pipe(Effect.exit); + expect(Exit.isFailure(result)).toBe(true); + }), + ), + ); + + it.live("copies through read-only source directories before preserving their modes", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "directory-copy-readonly-" }); + const source = path.join(root, "source"); + const nested = path.join(source, "nested"); + const destination = path.join(root, "destination"); + yield* fs.makeDirectory(nested, { recursive: true }); + yield* fs.writeFileString(path.join(nested, "child.txt"), "child\n"); + if (process.platform !== "win32") yield* fs.chmod(nested, 0o555); + + yield* copyDirectory(source, destination); + + expect(yield* fs.readFileString(path.join(destination, "nested", "child.txt"))).toBe( + "child\n", + ); + if (process.platform !== "win32") { + expect(Number((yield* fs.stat(path.join(destination, "nested"))).mode) & 0o777).toBe( + 0o555, + ); + yield* fs.chmod(nested, 0o755); + yield* fs.chmod(path.join(destination, "nested"), 0o755); + } + }), + ), + ); +}); diff --git a/packages/stack/src/storage/DirectoryCopy.ts b/packages/stack/src/storage/DirectoryCopy.ts new file mode 100644 index 0000000000..4af4d13253 --- /dev/null +++ b/packages/stack/src/storage/DirectoryCopy.ts @@ -0,0 +1,179 @@ +// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no lstat or clone-copy operation. +import { copyFile as nativeCopyFile, lstat } from "node:fs/promises"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no clone-copy flags. +import { constants as fsConstants } from "node:fs"; +import { Effect, FileSystem, Path, Schema } from "effect"; + +export class DirectoryCopyError extends Schema.TaggedError()( + "DirectoryCopyError", + { + operation: Schema.String, + source: Schema.String, + destination: Schema.String, + cause: Schema.Defect(), + }, +) {} + +class NativeFileError extends Schema.TaggedError()("NativeFileError", { + cause: Schema.Defect(), +}) {} + +type NativeStats = Awaited>; + +const errorFor = (operation: string, source: string, destination: string, cause: unknown) => + new DirectoryCopyError({ operation, source, destination, cause }); + +const isCloneUnsupported = (cause: unknown): boolean => { + if (Schema.is(NativeFileError)(cause)) return isCloneUnsupported(cause.cause); + if (typeof cause !== "object" || cause === null || !("code" in cause)) return false; + const code = cause.code; + return ( + code === "ENOTSUP" || + code === "EOPNOTSUPP" || + code === "ENOSYS" || + code === "EXDEV" || + code === "EINVAL" + ); +}; + +const isNotFound = (cause: unknown): boolean => + Schema.is(NativeFileError)(cause) + ? isNotFound(cause.cause) + : typeof cause === "object" && cause !== null && "code" in cause && cause.code === "ENOENT"; + +// FileSystem does not expose lstat or clone flags, so these two operations stay at the +// native boundary while the traversal and directory operations use the Effect service. +const nativeLstat = (target: string): Effect.Effect => + Effect.uninterruptible( + Effect.tryPromise({ + try: () => lstat(target), + catch: (cause) => new NativeFileError({ cause }), + }), + ); + +const nativeCopy = ( + source: string, + destination: string, + clone: boolean, +): Effect.Effect => + Effect.uninterruptible( + Effect.tryPromise({ + try: () => + nativeCopyFile(source, destination, clone ? fsConstants.COPYFILE_FICLONE_FORCE : 0), + catch: (cause) => new NativeFileError({ cause }), + }), + ); + +const inspect = ( + source: string, + destination: string, +): Effect.Effect => + nativeLstat(source).pipe( + Effect.mapError((cause) => errorFor("lstat", source, destination, cause)), + ); + +const copyFile = (source: string, destination: string): Effect.Effect => { + const regular = nativeCopy(source, destination, false).pipe( + Effect.mapError((cause) => errorFor("copyFile", source, destination, cause)), + ); + if (process.platform === "win32") return regular; + return nativeCopy(source, destination, true).pipe( + Effect.catch((cause: NativeFileError) => + isCloneUnsupported(cause) + ? regular + : Effect.fail(errorFor("copyFile", source, destination, cause)), + ), + ); +}; + +const validateTree = ( + source: string, + destination: string, + fs: FileSystem.FileSystem, + path: Path.Path, +): Effect.Effect => + Effect.gen(function* () { + const stats = yield* inspect(source, destination); + if (stats.isSymbolicLink()) + return yield* errorFor("validate", source, destination, "symbolic link"); + if (!stats.isDirectory()) + return yield* errorFor("validate", source, destination, "not a directory"); + const entries = yield* fs + .readDirectory(source) + .pipe(Effect.mapError((cause) => errorFor("readDirectory", source, destination, cause))); + for (const entry of entries) { + const childSource = path.join(source, entry); + const childDestination = path.join(destination, entry); + const childStats = yield* inspect(childSource, childDestination); + if (childStats.isSymbolicLink()) + return yield* errorFor("validate", childSource, childDestination, "symbolic link"); + if (childStats.isDirectory()) yield* validateTree(childSource, childDestination, fs, path); + else if (!childStats.isFile()) + return yield* errorFor("validate", childSource, childDestination, "special file"); + } + }); + +const copyTree = ( + source: string, + destination: string, + fs: FileSystem.FileSystem, + path: Path.Path, +): Effect.Effect => + Effect.gen(function* () { + const sourceStats = yield* inspect(source, destination); + if (sourceStats.isSymbolicLink()) + return yield* errorFor("validate", source, destination, "symbolic link"); + if (sourceStats.isDirectory() === false) + return yield* errorFor("validate", source, destination, "not a directory"); + + yield* fs + .makeDirectory(destination, { + mode: (Number(sourceStats.mode) | 0o700) & 0o7777, + }) + .pipe(Effect.mapError((cause) => errorFor("makeDirectory", source, destination, cause))); + + const entries = yield* fs + .readDirectory(source) + .pipe(Effect.mapError((cause) => errorFor("readDirectory", source, destination, cause))); + for (const entry of entries) { + const childSource = path.join(source, entry); + const childDestination = path.join(destination, entry); + const childStats = yield* inspect(childSource, destination); + if (childStats.isSymbolicLink()) + return yield* errorFor("validate", childSource, childDestination, "symbolic link"); + if (childStats.isDirectory()) { + yield* copyTree(childSource, childDestination, fs, path); + } else if (childStats.isFile()) { + yield* copyFile(childSource, childDestination); + yield* fs + .chmod(childDestination, Number(childStats.mode) & 0o7777) + .pipe( + Effect.mapError((cause) => errorFor("chmod", childSource, childDestination, cause)), + ); + } else { + return yield* errorFor("validate", childSource, childDestination, "special file"); + } + } + yield* fs + .chmod(destination, Number(sourceStats.mode) & 0o7777) + .pipe(Effect.mapError((cause) => errorFor("chmod", source, destination, cause))); + }); + +export const copyDirectory = Effect.fn("DirectoryCopy.copyDirectory")(function* ( + source: string, + destination: string, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* nativeLstat(destination).pipe( + Effect.matchEffect({ + onFailure: (cause) => + isNotFound(cause) + ? Effect.void + : Effect.fail(errorFor("lstat", source, destination, cause)), + onSuccess: () => Effect.fail(errorFor("validate", source, destination, "destination exists")), + }), + ); + yield* validateTree(source, destination, fs, path); + return yield* copyTree(source, destination, fs, path); +}); diff --git a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts new file mode 100644 index 0000000000..0a552d86d6 --- /dev/null +++ b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts @@ -0,0 +1,619 @@ +import { NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { + Crypto, + Data, + Effect, + Exit, + FileSystem, + Option, + Path, + Schema, + Scope, + Stream, +} from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { makeContainerRuntime } from "../runtime/Container.ts"; +import { makeDatabaseSnapshots } from "../services/DatabaseSnapshot.ts"; +import { makeDockerDatabaseStorage } from "./DockerDatabaseStorage.ts"; + +const helperImage = + "debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; +const Marker = Schema.Struct({ + backend: Schema.Literals(["docker", "host"]), + volume: Schema.optionalKey(Schema.String), + namespace: Schema.String, + cacheNamespace: Schema.String, + daemonId: Schema.optionalKey(Schema.String), + initialized: Schema.Boolean, +}); + +class DockerTestError extends Data.TaggedError("DockerTestError")<{ + readonly message: string; +}> {} + +const docker = Effect.fn("DockerStorageTest.docker")((args: ReadonlyArray) => + Effect.scoped( + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn(ChildProcess.make("docker", args, { stdin: "ignore" })); + const [stdout, stderr, code] = yield* Effect.all( + [ + child.stdout.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (all, chunk) => all + chunk, + ), + ), + child.stderr.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (all, chunk) => all + chunk, + ), + ), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + if (Number(code) !== 0) + return yield* new DockerTestError({ message: stderr.trim() || `docker exited ${code}` }); + return stdout.trim(); + }), + ), +); + +const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; + +describe("Docker database storage", { timeout: 120_000 }, () => { + it.live("round-trips stopped data and protects managed namespaces", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-" }); + const stateRoot = path.join(root, "state"); + const storageRoot = path.join(stateRoot, "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const sourceRoot = path.join(storageRoot, "source"); + const targetRoot = path.join(storageRoot, "target"); + yield* fs.makeDirectory(sourceRoot, { recursive: true }); + yield* fs.makeDirectory(targetRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ engine: "docker" }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const makeStorage = (instanceId: string, instanceRoot: string) => + makeDockerDatabaseStorage({ + runtime: "docker", + stackId: "storage-test", + instanceId, + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + const source = yield* makeStorage("source", sourceRoot); + const target = yield* makeStorage("target", targetRoot); + const ownerScope = yield* Scope.Scope; + yield* Scope.addFinalizer( + ownerScope, + Effect.gen(function* () { + yield* source.destroyData("17").pipe(Effect.ignore); + yield* target.destroyData("17").pipe(Effect.ignore); + const marker = yield* fs + .readFileString(path.join(sourceRoot, ".supabase-database-storage.json")) + .pipe(Effect.option); + if (Option.isSome(marker)) { + const parsed = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + marker.value, + ).pipe(Effect.option); + if (Option.isSome(parsed) && parsed.value.volume !== undefined) + yield* docker(["volume", "rm", parsed.value.volume]).pipe(Effect.ignore); + } + }).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.ignore, + ), + ); + yield* source.prepare("17"); + const sourceMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), + ); + if (sourceMarker.backend !== "docker" || sourceMarker.volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `set -eu; printf 17 > ${quote(`/store/${sourceMarker.namespace}/data/PG_VERSION`)}; printf source > ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}`, + ]); + yield* source.markInitialized("17"); + yield* fs.writeFileString( + path.join(sourceRoot, ".supabase-database-ready.json"), + '{"version":"17","runtime":"docker","profile":"supabase"}', + ); + yield* source.saveSnapshot("17", "roundtrip"); + + yield* target.prepare("17"); + expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(true); + const targetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(targetRoot, ".supabase-database-storage.json")), + ); + expect(targetMarker.volume).toBe(sourceMarker.volume); + yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store,volume-subpath=${targetMarker.namespace}/data`, + helperImage, + "/bin/sh", + "-c", + `test "$(cat /store/fixture)" = source`, + ]); + + yield* target.removeData("17"); + const resetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(targetRoot, ".supabase-database-storage.json")), + ); + expect(resetMarker.volume).toBe(sourceMarker.volume); + expect(resetMarker.initialized).toBe(false); + + yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + 'descriptor=$(find /store -name descriptor.json -print -quit); printf corrupt > "$descriptor"', + ]); + const corrupt = yield* target.restoreSnapshot("17", "roundtrip").pipe( + Effect.matchEffect({ + onFailure: () => Effect.succeed(true), + onSuccess: () => Effect.succeed(false), + }), + ); + expect(corrupt).toBe(true); + yield* source.saveSnapshot("17", "roundtrip"); + + yield* target.restoreSnapshot("17", "roundtrip"); + yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `rm -rf ${quote(`/store/${targetMarker.namespace}/data`)}`, + ]); + const missing = yield* target.prepare("17").pipe(Effect.exit); + expect(missing).toSatisfy((exit) => Exit.isFailure(exit)); + + yield* target.removeData("17"); + yield* source.destroyData("17"); + expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(true); + yield* target.destroyData("17"); + yield* docker(["volume", "rm", sourceMarker.volume]); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("removes an unprepared storage without requiring Docker", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-empty-" }); + const instanceRoot = path.join(root, "state", "stack", "data", "empty"); + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + stackId: "storage-empty-test", + instanceId: "empty", + instanceRoot, + root: path.join(root, "state", "stack", "data"), + cacheRoot: path.join(root, "cache"), + fs, + path, + crypto, + container: undefined, + spawner, + }); + yield* storage.removeData("17"); + yield* storage.destroyData("17"); + expect(yield* fs.exists(path.join(instanceRoot, ".supabase-database-storage.json"))).toBe( + false, + ); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("destroys legacy host data without a storage marker", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-legacy-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "legacy"); + const dataRoot = path.join(instanceRoot, "data"); + yield* fs.makeDirectory(dataRoot, { recursive: true }); + yield* fs.writeFileString(path.join(dataRoot, "PG_VERSION"), "17\n"); + yield* fs.writeFileString(path.join(dataRoot, "fixture"), "legacy"); + yield* fs.writeFileString( + path.join(instanceRoot, ".supabase-database-ready.json"), + '{"version":"17","runtime":"docker","profile":"supabase"}', + ); + const container = yield* makeContainerRuntime({ engine: "docker" }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + yield* docker([ + "run", + "--rm", + "--mount", + `type=bind,src=${instanceRoot},dst=/instance`, + helperImage, + "/bin/sh", + "-c", + "chown -R 100:101 /instance/data; chmod 700 /instance/data", + ]); + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + stackId: `storage-legacy-${yield* crypto.randomUUIDv4}`, + instanceId: "legacy", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + expect(yield* fs.exists(path.join(instanceRoot, ".supabase-database-storage.json"))).toBe( + false, + ); + yield* storage.destroyData("17"); + expect(yield* fs.exists(dataRoot)).toBe(false); + expect(yield* fs.exists(path.join(instanceRoot, ".supabase-database-ready.json"))).toBe( + false, + ); + yield* fs.remove(cacheRoot, { recursive: true, force: true }); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("recovers after an owned helper is removed externally", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-helper-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "recovery"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ engine: "docker" }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-helper-recovery-${yield* crypto.randomUUIDv4}`; + const instanceId = "recovery"; + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + stackId, + instanceId, + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + yield* storage.prepare("17"); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(instanceRoot, ".supabase-database-storage.json")), + ); + const volume = marker.volume; + if (volume === undefined) + return yield* new DockerTestError({ message: "Docker storage volume missing" }); + const helpers = yield* docker([ + "ps", + "--filter", + "label=com.supabase.stack=" + stackId, + "--filter", + "label=com.supabase.instance=" + instanceId, + "--format", + "{{.Names}}", + ]); + const helper = helpers.split("\n").find((name) => name.length > 0); + if (helper === undefined) + return yield* new DockerTestError({ message: "Owned helper was not discoverable" }); + yield* docker(["rm", "-f", helper]); + const failed = yield* storage.prepare("17").pipe(Effect.exit); + expect(failed).toSatisfy((exit) => Exit.isFailure(exit)); + yield* storage.prepare("17"); + yield* storage.destroyData("17"); + yield* docker(["volume", "rm", volume]); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("destroys an initialized storage after its volume is removed", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-missing-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "missing"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ engine: "docker" }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-missing-${yield* crypto.randomUUIDv4}`; + const makeStorage = () => + makeDockerDatabaseStorage({ + runtime: "docker", + stackId, + instanceId: "missing", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + let volume: string | undefined; + yield* Effect.scoped( + Effect.gen(function* () { + const storage = yield* makeStorage(); + yield* storage.prepare("17"); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(instanceRoot, ".supabase-database-storage.json")), + ); + volume = marker.volume; + if (volume === undefined) + return yield* new DockerTestError({ message: "Docker storage volume missing" }); + const mount = yield* storage.mount("17"); + if (mount.type !== "volume" || mount.volumeSubpath === undefined) + return yield* new DockerTestError({ message: "Docker volume backend unavailable" }); + yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${volume},dst=/data,volume-subpath=${mount.volumeSubpath}`, + helperImage, + "/bin/sh", + "-c", + "printf 17 > /data/PG_VERSION", + ]); + yield* storage.markInitialized("17"); + }), + ); + if (volume === undefined) return yield* new DockerTestError({ message: "Missing volume" }); + yield* docker(["volume", "rm", volume]); + yield* Effect.scoped( + Effect.gen(function* () { + const storage = yield* makeStorage(); + const prepare = yield* storage.prepare("17").pipe(Effect.exit); + expect(prepare).toSatisfy((exit) => Exit.isFailure(exit)); + expect(yield* storage.destroyData("17").pipe(Effect.exit)).toSatisfy((exit) => + Exit.isSuccess(exit), + ); + }), + ); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("adopts root-owned host data through helper operations", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-host-" }); + const stateRoot = path.join(root, "state"); + const storageRoot = path.join(stateRoot, "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "adopted"); + const dataRoot = path.join(instanceRoot, "data"); + yield* fs.makeDirectory(dataRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + yield* fs.writeFileString(path.join(dataRoot, "PG_VERSION"), "17\n"); + yield* fs.writeFileString(path.join(dataRoot, "fixture"), "adopted"); + yield* fs.writeFileString( + path.join(instanceRoot, ".supabase-database-ready.json"), + '{"version":"17","runtime":"docker","profile":"supabase"}', + ); + const container = yield* makeContainerRuntime({ engine: "docker" }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + yield* docker([ + "run", + "--rm", + "--mount", + `type=bind,src=${instanceRoot},dst=/instance`, + helperImage, + "/bin/sh", + "-c", + "chown -R 100:101 /instance/data; chmod 700 /instance/data", + ]); + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + stackId: "storage-host-test", + instanceId: "adopted", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + yield* storage.prepare("17"); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(instanceRoot, ".supabase-database-storage.json")), + ); + expect(marker.backend).toBe("host"); + expect(marker.initialized).toBe(true); + yield* storage.saveSnapshot("17", "adopted"); + const nativeRoot = path.join(root, "native"); + yield* fs.makeDirectory(path.join(nativeRoot, "data"), { recursive: true }); + yield* fs.writeFileString(path.join(nativeRoot, "data", "PG_VERSION"), "17\n"); + yield* fs.writeFileString( + path.join(nativeRoot, ".supabase-database-ready.json"), + '{"version":"17.6.1.173","runtime":"native","profile":"supabase"}', + ); + const nativeSnapshots = yield* makeDatabaseSnapshots({ + instanceRoot: nativeRoot, + cacheRoot, + runtime: "native", + version: "17", + stackId: "native-interoperability", + instanceId: "database", + }); + yield* nativeSnapshots.saveSnapshot("native"); + yield* fs.remove(path.join(nativeRoot, "data"), { recursive: true }); + expect(yield* nativeSnapshots.restoreSnapshot("native")).toBe(true); + yield* storage.removeData("17"); + expect(yield* storage.restoreSnapshot("17", "adopted")).toBe(true); + yield* storage.removeData("17"); + yield* storage.destroyData("17"); + expect(yield* fs.exists(path.join(instanceRoot, ".supabase-database-storage.json"))).toBe( + true, + ); + expect(yield* fs.exists(dataRoot)).toBe(false); + yield* docker([ + "run", + "--rm", + "--mount", + `type=bind,src=${cacheRoot},dst=/cache`, + helperImage, + "/bin/sh", + "-c", + "rm -rf /cache/*", + ]); + yield* fs.remove(cacheRoot, { recursive: true, force: true }); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("reopens managed storage and preserves snapshots after source destruction", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-reopen-" }); + const stateRoot = path.join(root, "state"); + const storageRoot = path.join(stateRoot, "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const sourceRoot = path.join(storageRoot, "source"); + const targetRoot = path.join(storageRoot, "target"); + yield* fs.makeDirectory(sourceRoot, { recursive: true }); + yield* fs.makeDirectory(targetRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ engine: "docker" }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const makeStorage = (instanceId: string, instanceRoot: string) => + makeDockerDatabaseStorage({ + runtime: "docker", + stackId: "storage-reopen-test", + instanceId, + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + let sourceVolume: string | undefined; + yield* Effect.scoped( + Effect.gen(function* () { + const source = yield* makeStorage("source", sourceRoot); + yield* source.prepare("17"); + const mount = yield* source.mount("17"); + if (mount.type !== "volume" || mount.volumeSubpath === undefined) + return yield* new DockerTestError({ message: "Docker volume backend unavailable" }); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), + ); + sourceVolume = marker.volume; + if (sourceVolume === undefined) + return yield* new DockerTestError({ message: "Docker storage volume missing" }); + yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceVolume},dst=/data,volume-subpath=${mount.volumeSubpath}`, + helperImage, + "/bin/sh", + "-c", + "printf 17 > /data/PG_VERSION; printf reopened > /data/fixture", + ]); + yield* source.markInitialized("17"); + yield* fs.writeFileString( + path.join(sourceRoot, ".supabase-database-ready.json"), + '{"version":"17","runtime":"docker","profile":"supabase"}', + ); + yield* source.saveSnapshot("17", "reopened"); + }), + ); + yield* Effect.scoped( + Effect.gen(function* () { + const source = yield* makeStorage("source", sourceRoot); + yield* source.prepare("17"); + yield* source.saveSnapshot("17", "reopened"); + yield* source.removeData("17"); + expect(yield* source.restoreSnapshot("17", "reopened")).toBe(true); + yield* source.saveSnapshot("17", "reopened-again"); + yield* source.destroyData("17"); + const target = yield* makeStorage("target", targetRoot); + yield* target.prepare("17"); + expect(yield* target.restoreSnapshot("17", "reopened-again")).toBe(true); + const targetMount = yield* target.mount("17"); + if (targetMount.type !== "volume" || targetMount.volumeSubpath === undefined) + return yield* new DockerTestError({ message: "Docker target volume unavailable" }); + yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceVolume ?? ""},dst=/data,volume-subpath=${targetMount.volumeSubpath}`, + helperImage, + "/bin/sh", + "-c", + 'test "$(cat /data/fixture)" = reopened', + ]); + yield* target.destroyData("17"); + }), + ); + if (sourceVolume !== undefined) yield* docker(["volume", "rm", sourceVolume]); + yield* fs.remove(cacheRoot, { recursive: true, force: true }); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); +}); diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts new file mode 100644 index 0000000000..cae4ab01b0 --- /dev/null +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -0,0 +1,815 @@ +import { + Crypto, + Effect, + Exit, + FileSystem, + Option, + Path, + Ref, + Schema, + Scope, + Semaphore, + Stream, +} from "effect"; +import { ChildProcess } from "effect/unstable/process"; +import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; +import type { ContainerRuntime } from "../runtime/Container.ts"; +import type { DatabaseRuntime } from "../services/Database.ts"; + +const HELPER_IMAGE = + "docker.io/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; + +const Marker = Schema.Struct({ + backend: Schema.Literals(["docker", "host"]), + volume: Schema.optionalKey(Schema.String), + namespace: Schema.String, + cacheNamespace: Schema.String, + daemonId: Schema.optionalKey(Schema.String), + initialized: Schema.Boolean, +}); +type Marker = Schema.Schema.Type; +const SnapshotIdentity = Schema.Struct({ + format: Schema.String, + version: Schema.String, + runtime: Schema.String, + platform: Schema.String, + arch: Schema.String, + profile: Schema.String, + key: Schema.String, +}); +const SnapshotDescriptor = Schema.Struct({ + format: Schema.String, + version: Schema.String, + runtime: Schema.String, + platform: Schema.String, + arch: Schema.String, + profile: Schema.String, + keyDigest: Schema.String, +}); +const ReadyMarker = Schema.Struct({ + version: Schema.String, + runtime: Schema.Literals(["native", "docker", "podman"]), + profile: Schema.Literal("supabase"), +}); + +export class DockerDatabaseStorageError extends Schema.TaggedError()( + "DockerDatabaseStorageError", + { operation: Schema.String, message: Schema.String, cause: Schema.optionalKey(Schema.Defect()) }, +) {} + +export interface DatabaseStorageMount { + readonly source: string; + readonly target: string; + readonly readOnly: boolean; + readonly type?: "bind" | "volume"; + readonly volumeSubpath?: string; +} + +export interface DockerDatabaseStorage { + readonly prepare: (version: string) => Effect.Effect; + readonly needsDataChown: Effect.Effect; + readonly mount: ( + version: string, + ) => Effect.Effect; + readonly markInitialized: (version: string) => Effect.Effect; + readonly removeData: (version: string) => Effect.Effect; + readonly destroyData: (version: string) => Effect.Effect; + readonly saveSnapshot: ( + version: string, + key: string, + ) => Effect.Effect; + readonly restoreSnapshot: ( + version: string, + key: string, + ) => Effect.Effect; +} + +const errorFor = (operation: string, cause: unknown) => + Schema.is(DockerDatabaseStorageError)(cause) + ? cause + : new DockerDatabaseStorageError({ + operation, + message: cause instanceof Error ? cause.message : String(cause), + cause, + }); + +const shellQuote = (value: string): string => `'${value.replaceAll("'", "'\\''")}'`; + +const parseMajor = (version: string): number | undefined => { + const major = Number(version.trim().split(".")[0]); + return Number.isInteger(major) ? major : undefined; +}; + +/** Owns the placement and lifecycle of one database's Docker data and snapshot namespaces. */ +export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make")( + (options: { + readonly runtime: DatabaseRuntime; + readonly stackId: string; + readonly instanceId: string; + readonly instanceRoot: string; + readonly root: string; + readonly cacheRoot: string; + readonly fs: FileSystem.FileSystem; + readonly path: Path.Path; + readonly crypto: Crypto.Crypto; + readonly container: ContainerRuntime | undefined; + readonly spawner: ChildProcessSpawnerService["Service"]; + }): Effect.Effect => + Effect.gen(function* () { + const markerPath = options.path.join(options.instanceRoot, ".supabase-database-storage.json"); + const stateRoot = options.path.dirname(options.path.dirname(options.root)); + const dataNamespace = `instance-${options.stackId}-${options.instanceId}`; + const hash = (value: string) => + options.crypto.digest("SHA-256", new TextEncoder().encode(value)).pipe( + Effect.map((bytes) => + Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""), + ), + Effect.mapError((cause) => errorFor("identity", cause)), + ); + const encodeMarker = Schema.encodeEffect(Schema.fromJsonString(Marker)); + const encodeIdentity = Schema.encodeEffect(Schema.fromJsonString(SnapshotIdentity)); + const encodeDescriptor = Schema.encodeEffect(Schema.fromJsonString(SnapshotDescriptor)); + const validateMarker = (marker: Marker) => + Effect.gen(function* () { + if ( + marker.namespace !== dataNamespace || + !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/u.test(marker.namespace) || + !/^cache-[a-f0-9]{32}$/u.test(marker.cacheNamespace) || + (marker.volume !== undefined && + !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/u.test(marker.volume)) + ) + return yield* errorFor("data", "Recorded database storage identity is invalid"); + return marker; + }); + const selectedCache = yield* Ref.make(undefined); + const selectionLock = yield* Semaphore.make(1); + const selected = selectionLock.withPermit( + Effect.gen(function* () { + const cached = yield* Ref.get(selectedCache); + if (cached !== undefined) return cached; + const value = yield* Effect.gen(function* () { + if (options.runtime === "native" || options.container === undefined) + return { + backend: "host" as const, + namespace: dataNamespace, + cacheNamespace: "native", + initialized: false, + }; + if (options.runtime !== "docker") { + const present = yield* options.fs + .exists(markerPath) + .pipe(Effect.mapError((cause) => errorFor("marker", cause))); + if (present) + return yield* options.fs.readFileString(markerPath).pipe( + Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(Marker))), + Effect.flatMap(validateMarker), + Effect.mapError((cause) => errorFor("marker", cause)), + ); + yield* options.fs + .makeDirectory(options.cacheRoot, { recursive: true }) + .pipe(Effect.mapError((cause) => errorFor("identity", cause))); + const cacheRoot = yield* options.fs + .realPath(options.cacheRoot) + .pipe(Effect.mapError((cause) => errorFor("identity", cause))); + const hostData = options.path.join(options.instanceRoot, "data"); + const initialized = + (yield* options.fs + .exists(hostData) + .pipe(Effect.mapError((cause) => errorFor("data", cause)))) && + (yield* options.fs.readDirectory(hostData).pipe( + Effect.map((entries) => entries.length > 0), + Effect.orElseSucceed(() => true), + )); + const value: Marker = { + backend: "host", + namespace: dataNamespace, + cacheNamespace: `cache-${(yield* hash(cacheRoot)).slice(0, 32)}`, + initialized, + }; + const encoded = yield* encodeMarker(value); + yield* options.fs + .writeFileString(markerPath, encoded, { mode: 0o600 }) + .pipe(Effect.mapError((cause) => errorFor("marker", cause))); + return value; + } + const daemonId = yield* engineCommand(["info", "--format", "{{.ID}}"]); + const version = yield* engineCommand([ + "version", + "--format", + "{{.Client.Version}}|{{.Server.Version}}", + ]); + const [clientVersion, serverVersion] = version.split("|"); + const clientMajor = parseMajor(clientVersion ?? ""); + const serverMajor = parseMajor(serverVersion ?? ""); + if (clientMajor === undefined || serverMajor === undefined) + return yield* errorFor("engine", "Docker returned an invalid version"); + const canonicalStateRoot = yield* options.fs + .realPath(stateRoot) + .pipe(Effect.mapError((cause) => errorFor("identity", cause))); + yield* options.fs + .makeDirectory(options.cacheRoot, { recursive: true }) + .pipe(Effect.mapError((cause) => errorFor("identity", cause))); + const canonicalCacheRoot = yield* options.fs + .realPath(options.cacheRoot) + .pipe(Effect.mapError((cause) => errorFor("identity", cause))); + const stateDigest = yield* hash(`${canonicalStateRoot}\0${daemonId.trim()}`); + const cacheDigest = yield* hash(canonicalCacheRoot); + const volume = `supabase-db-${stateDigest.slice(0, 32)}`; + const markerPresent = yield* options.fs + .exists(markerPath) + .pipe(Effect.mapError((cause) => errorFor("marker", cause))); + const marker = markerPresent + ? Option.some( + yield* options.fs.readFileString(markerPath).pipe( + Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(Marker))), + Effect.mapError((cause) => errorFor("marker", cause)), + ), + ) + : Option.none(); + if (Option.isSome(marker)) { + const validMarker = yield* validateMarker(marker.value); + if (marker.value.backend === "docker") { + if (validMarker.daemonId !== daemonId.trim()) + return yield* errorFor( + "data", + "Recorded Docker database storage does not match this daemon", + ); + if (validMarker.volume === undefined) + return yield* errorFor("data", "Recorded Docker storage volume is missing"); + const volume = validMarker.volume; + yield* engineCommand(["volume", "inspect", volume]).pipe( + Effect.catchTag("DockerDatabaseStorageError", (cause) => + !validMarker.initialized && /(?:no such volume|not found)/iu.test(cause.message) + ? engineCommand([ + "volume", + "create", + "--label", + "com.supabase.stack-managed=true", + "--label", + `com.supabase.stack-state-root=${stateDigest}`, + volume, + ]).pipe(Effect.asVoid) + : Effect.fail(cause), + ), + ); + return validMarker; + } + return validMarker; + } + const hostData = options.path.join(options.instanceRoot, "data"); + if ( + yield* options.fs + .exists(hostData) + .pipe(Effect.mapError((cause) => errorFor("data", cause))) + ) { + const entries = yield* options.fs.readDirectory(hostData).pipe( + // A legacy data directory may be owned by PostgreSQL's container UID. + // Let the root helper validate and adopt it instead of treating EACCES as empty. + Effect.orElseSucceed(() => ["inaccessible-data"]), + ); + if (entries.length > 0) { + const value: Marker = { + backend: "host", + namespace: dataNamespace, + cacheNamespace: `cache-${cacheDigest.slice(0, 32)}`, + initialized: true, + }; + const encoded = yield* encodeMarker(value); + yield* options.fs + .writeFileString(markerPath, encoded, { mode: 0o600 }) + .pipe(Effect.mapError((cause) => errorFor("marker", cause))); + return value; + } + } + // Docker 26 introduced volume-subpath. Older engines retain the host-backed path. + const backend = clientMajor >= 26 && serverMajor >= 26 ? "docker" : "host"; + const value: Marker = { + backend, + ...(backend === "docker" ? { volume, daemonId: daemonId.trim() } : {}), + namespace: dataNamespace, + cacheNamespace: `cache-${cacheDigest.slice(0, 32)}`, + initialized: false, + }; + if (backend === "docker") + yield* engineCommand([ + "volume", + "create", + "--label", + "com.supabase.stack-managed=true", + "--label", + `com.supabase.stack-state-root=${stateDigest}`, + volume, + ]); + yield* options.fs + .writeFileString(markerPath, yield* encodeMarker(value), { mode: 0o600 }) + .pipe(Effect.mapError((cause) => errorFor("marker", cause))); + return value; + }); + yield* Ref.set(selectedCache, value); + return value; + }), + ); + + const engineCommand = Effect.fn("DockerDatabaseStorage.engineCommand")( + (args: ReadonlyArray): Effect.Effect => + Effect.scoped( + Effect.gen(function* () { + const child = yield* options.spawner + .spawn(ChildProcess.make(options.runtime, args, { stdin: "ignore" })) + .pipe(Effect.mapError((cause) => errorFor("engine", cause))); + const [stdout, stderr, code] = yield* Effect.all( + [ + child.stdout.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (all: string, chunk: string) => all + chunk, + ), + ), + child.stderr.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (all: string, chunk: string) => all + chunk, + ), + ), + child.exitCode.pipe(Effect.mapError((cause) => errorFor("engine", cause))), + ], + { concurrency: "unbounded" }, + ); + if (Number(code) !== 0) + return yield* errorFor( + "engine", + stderr.trim() || `Container engine exited with ${code}`, + ); + return stdout.trim(); + }), + ).pipe(Effect.mapError((cause) => errorFor("engine", cause))), + ); + + const helperId = yield* Ref.make(undefined); + const helperCleanupPending = yield* Ref.make(false); + const operationLock = yield* Semaphore.make(1); + const ownerScope = yield* Scope.Scope; + const mountField = (key: string, value: string) => { + const field = `${key}=${value}`; + return /[",\n\r]/u.test(field) ? `"${field.replaceAll('"', '""')}"` : field; + }; + const mountArgs = (mounts: ReadonlyArray) => + mounts.flatMap((mount) => [ + "--mount", + [ + `type=${mount.type ?? "bind"}`, + mountField("src", mount.source), + mountField("dst", mount.target), + ...(mount.volumeSubpath === undefined + ? [] + : [mountField("volume-subpath", mount.volumeSubpath)]), + ...(mount.readOnly ? ["ro"] : []), + ].join(","), + ]); + const removeHelper = Effect.fn("DockerDatabaseStorage.removeHelper")(() => + Effect.uninterruptible( + Effect.gen(function* () { + const current = yield* Ref.get(helperId); + if (current !== undefined) { + // Keep the owned identity until the remote container is gone. + yield* engineCommand(["rm", "-f", current]).pipe( + Effect.catchTag("DockerDatabaseStorageError", (cause) => + /no such container/iu.test(cause.message) ? Effect.void : Effect.fail(cause), + ), + ); + yield* Ref.set(helperId, undefined); + yield* Ref.set(helperCleanupPending, false); + } + }), + ), + ); + yield* Scope.addFinalizer( + ownerScope, + removeHelper().pipe(Effect.catch((cause) => Effect.logError(cause))), + ); + const acquireHelper = Effect.fn("DockerDatabaseStorage.acquireHelper")( + (mounts: ReadonlyArray) => + Effect.gen(function* () { + const current = yield* Ref.get(helperId); + if (current !== undefined && (yield* Ref.get(helperCleanupPending))) + yield* removeHelper(); + const active = yield* Ref.get(helperId); + if (active !== undefined) return active; + if (options.container === undefined) + return yield* errorFor("helper", "Container runtime is unavailable"); + if ( + mounts.some( + (mount) => mount.source === options.cacheRoot && mount.target === "/cache", + ) + ) + yield* options.fs + .makeDirectory(options.cacheRoot, { recursive: true }) + .pipe(Effect.mapError((cause) => errorFor("helper", cause))); + yield* options.container.prepare(HELPER_IMAGE); + const token = yield* options.crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => errorFor("helper", cause)), + ); + const name = `supabase-db-helper-${token}`; + // Register the deterministic owned name before the remote create starts so an + // interrupted docker run can still be removed by the same scope. + yield* Ref.set(helperId, name); + const created = yield* engineCommand([ + "run", + "-d", + "--name", + name, + "--label", + "com.supabase.stack-managed=true", + "--label", + `com.supabase.stack=${options.stackId}`, + "--label", + `com.supabase.instance=${options.instanceId}`, + ...mountArgs(mounts), + HELPER_IMAGE, + "/bin/sh", + "-c", + "trap : TERM INT; while :; do sleep 3600; done", + ]); + if (!/^[a-f0-9]{12,64}$/u.test(created)) + return yield* errorFor("helper", "Docker returned an invalid helper identity"); + return name; + }), + ); + + const runHelper = Effect.fn("DockerDatabaseStorage.helper")(( + command: string, + mounts: ReadonlyArray, + ): Effect.Effect => { + const cleanupAfterFailure = Effect.gen(function* () { + if ((yield* Ref.get(helperId)) === undefined) return; + yield* Ref.set(helperCleanupPending, true); + yield* removeHelper(); + }); + const operation = Effect.uninterruptibleMask((restore) => + restore( + Effect.gen(function* () { + const id = yield* acquireHelper(mounts); + return yield* engineCommand(["exec", id, "/bin/sh", "-c", command]); + }), + ).pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) ? Effect.uninterruptible(cleanupAfterFailure) : Effect.void, + ), + Effect.mapError((cause) => errorFor("mount", cause)), + ), + ); + return operationLock + .withPermit(operation) + .pipe(Effect.mapError((cause) => errorFor("helper", cause))); + }); + + const getMarker = Effect.gen(function* () { + const marker = yield* options.fs + .readFileString(markerPath) + .pipe(Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(Marker)))); + return yield* validateMarker(marker); + }); + const getMarkerIfPresent = Effect.gen(function* () { + const present = yield* options.fs + .exists(markerPath) + .pipe(Effect.mapError((cause) => errorFor("marker", cause))); + return present ? Option.some(yield* getMarker) : Option.none(); + }); + const getMarkerForRemoval = Effect.gen(function* () { + const existing = yield* getMarkerIfPresent; + if (Option.isSome(existing)) { + if (options.runtime === "docker") { + yield* selected; + return Option.some(yield* getMarker); + } + return existing; + } + const data = options.path.join(options.instanceRoot, "data"); + if (!(yield* options.fs.exists(data))) return Option.none(); + const nonEmpty = yield* options.fs.readDirectory(data).pipe( + Effect.map((entries) => entries.length > 0), + Effect.orElseSucceed(() => true), + ); + if (!nonEmpty) return Option.none(); + yield* selected; + return Option.some(yield* getMarker); + }); + const writeMarker = (marker: Marker) => + encodeMarker(marker).pipe( + Effect.flatMap((encoded) => + options.fs.writeFileString(markerPath, encoded, { mode: 0o600 }), + ), + ); + const publishReadyMarker = (version: string) => + Effect.scoped( + Effect.gen(function* () { + const stage = yield* options.fs.makeTempDirectoryScoped({ + directory: options.instanceRoot, + prefix: ".ready-", + }); + const ready = yield* Schema.encodeEffect(Schema.fromJsonString(ReadyMarker))({ + version, + runtime: options.runtime, + profile: "supabase", + }); + const staged = options.path.join(stage, "marker"); + const destination = options.path.join( + options.instanceRoot, + ".supabase-database-ready.json", + ); + yield* options.fs.writeFileString(staged, ready, { mode: 0o600 }); + yield* options.fs.rename(staged, destination); + }), + ); + const setup = Effect.fn("DockerDatabaseStorage.prepare")((version: string) => + Effect.gen(function* () { + yield* selected; + const marker = yield* getMarker; + if (marker.backend === "host") { + const data = options.path.join(options.instanceRoot, "data"); + if (marker.initialized) { + if (!(yield* options.fs.exists(data))) + return yield* errorFor("prepare", "Initialized database data is missing"); + const major = yield* runHelper( + "set -eu; if [ ! -f /instance/data/PG_VERSION ]; then echo 'Initialized PostgreSQL data is missing PG_VERSION' >&2; exit 1; fi; cat /instance/data/PG_VERSION", + [ + { source: options.instanceRoot, target: "/instance", readOnly: false }, + { source: options.cacheRoot, target: "/cache", readOnly: false }, + ], + ); + if (major.trim() !== majorVersion(version)) + return yield* errorFor( + "prepare", + "Initialized PostgreSQL major does not match the requested configuration", + ); + } else { + yield* options.fs.makeDirectory(data, { recursive: true, mode: 0o700 }); + } + return; + } + const store = `/store/${marker.namespace}`; + const cache = `/store/${marker.cacheNamespace}`; + if (marker.initialized) { + const major = yield* runHelper( + `set -eu; if [ ! -f ${shellQuote(`${store}/data/PG_VERSION`)} ]; then echo 'Initialized PostgreSQL data is missing PG_VERSION' >&2; exit 1; fi; cat ${shellQuote(`${store}/data/PG_VERSION`)}`, + [{ source: marker.volume ?? "", target: "/store", readOnly: false, type: "volume" }], + ); + if (major.trim() !== majorVersion(version)) + return yield* errorFor( + "prepare", + "Initialized PostgreSQL major does not match the requested configuration", + ); + } else { + yield* runHelper( + `set -eu; mkdir -p ${shellQuote(`${store}/data`)} ${shellQuote(`${cache}/entries`)} ${shellQuote(`${cache}/stages`)}; chown -R 100:101 ${shellQuote(store)}`, + [{ source: marker.volume ?? "", target: "/store", readOnly: false, type: "volume" }], + ); + } + }).pipe(Effect.mapError((cause) => errorFor("prepare", cause))), + ); + const majorVersion = (version: string) => version.split(".")[0] ?? version; + + const mount = (_version: string) => + selected.pipe( + Effect.flatMap(() => getMarker), + Effect.map((marker) => + marker.backend === "docker" + ? { + source: marker.volume ?? "", + target: "/var/lib/postgresql/data", + readOnly: false, + type: "volume" as const, + volumeSubpath: `${marker.namespace}/data`, + } + : { + source: options.path.join(options.instanceRoot, "data"), + target: "/var/lib/postgresql/data", + readOnly: false, + }, + ), + Effect.mapError((cause) => errorFor("mount", cause)), + ); + const needsDataChown = getMarker.pipe( + Effect.map((marker) => marker.backend === "host"), + Effect.mapError((cause) => errorFor("marker", cause)), + ); + const markInitialized = Effect.fn("DockerDatabaseStorage.markInitialized")( + (version: string) => + Effect.gen(function* () { + const marker = yield* getMarker; + if (marker.backend === "docker") { + yield* runHelper( + `set -eu; if [ ! -f ${shellQuote(`/store/${marker.namespace}/data/PG_VERSION`)} ]; then echo 'Database readiness requires PG_VERSION' >&2; exit 1; fi; actual=$(cat ${shellQuote(`/store/${marker.namespace}/data/PG_VERSION`)}); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Database PostgreSQL major does not match requested version' >&2; exit 1; fi`, + [ + { + source: marker.volume ?? "", + target: "/store", + readOnly: false, + type: "volume", + }, + ], + ); + } + yield* writeMarker({ ...marker, initialized: true }); + }).pipe(Effect.mapError((cause) => errorFor("ready", cause))), + ); + const removeData = Effect.fn("DockerDatabaseStorage.removeData")((_version: string) => + Effect.gen(function* () { + const markerOption = yield* getMarkerForRemoval; + if (Option.isNone(markerOption)) return; + const marker = markerOption.value; + if (marker.backend === "host") { + yield* runHelper( + `set -eu; rm -rf /instance/data /instance/.supabase-restore-*; mkdir -p /instance/data; chown 100:101 /instance/data`, + snapshotPaths(marker).mounts, + ); + } else { + yield* runHelper( + `set -eu; rm -rf ${shellQuote(`/store/${marker.namespace}/data`)}; mkdir -p ${shellQuote(`/store/${marker.namespace}/data`)}; chown 100:101 ${shellQuote(`/store/${marker.namespace}/data`)}`, + [{ source: marker.volume ?? "", target: "/store", readOnly: false, type: "volume" }], + ); + } + yield* options.fs + .remove(options.path.join(options.instanceRoot, ".supabase-database-ready.json"), { + force: true, + }) + .pipe(Effect.mapError((cause) => errorFor("reset", cause))); + yield* writeMarker({ ...marker, initialized: false }); + }).pipe(Effect.mapError((cause) => errorFor("reset", cause))), + ); + const destroyData = Effect.fn("DockerDatabaseStorage.destroyData")((_version: string) => + Effect.gen(function* () { + const present = yield* getMarkerIfPresent; + const markerOption = Option.isSome(present) ? present : yield* getMarkerForRemoval; + if (Option.isNone(markerOption)) return; + const marker = markerOption.value; + if (marker.backend === "host") { + yield* runHelper( + `set -eu; rm -rf /instance/data /instance/.supabase-restore-* /instance/.supabase-database-ready.json`, + snapshotPaths(marker).mounts, + ); + yield* removeHelper(); + } else { + const daemonId = yield* engineCommand(["info", "--format", "{{.ID}}"]).pipe( + Effect.map((value) => value.trim()), + ); + if (marker.daemonId !== daemonId) + return yield* errorFor( + "destroy", + "Recorded Docker database storage belongs to another daemon; switch back to the original Docker context before destroying it", + ); + if (marker.volume === undefined) + return yield* errorFor("destroy", "Recorded Docker storage volume is missing"); + const volumeExists = yield* engineCommand(["volume", "inspect", marker.volume]).pipe( + Effect.as(true), + Effect.catchTag("DockerDatabaseStorageError", (cause) => + /no such volume/iu.test(cause.message) ? Effect.succeed(false) : Effect.fail(cause), + ), + ); + if (volumeExists) + yield* runHelper(`set -eu; rm -rf ${shellQuote(`/store/${marker.namespace}`)}`, [ + { source: marker.volume, target: "/store", readOnly: false, type: "volume" }, + ]); + yield* removeHelper(); + } + }).pipe(Effect.mapError((cause) => errorFor("destroy", cause))), + ); + const snapshotPaths = (marker: Marker) => + marker.backend === "docker" + ? { + root: `/store/${marker.cacheNamespace}`, + source: `/store/${marker.namespace}/data`, + mounts: [ + { + source: marker.volume ?? "", + target: "/store", + readOnly: false, + type: "volume" as const, + }, + ], + } + : { + root: `/cache/stack-database-snapshots-helper/${marker.cacheNamespace}`, + source: "/instance/data", + mounts: [ + { source: options.instanceRoot, target: "/instance", readOnly: false as const }, + { source: options.cacheRoot, target: "/cache", readOnly: false as const }, + ], + }; + const descriptorDigest = (version: string, key: string) => + encodeIdentity({ + format: "supabase-database-snapshot-v1", + version, + runtime: options.runtime, + platform: process.platform, + arch: process.arch, + profile: "supabase", + key, + }).pipe( + Effect.mapError((cause) => errorFor("snapshot", cause)), + Effect.flatMap(hash), + ); + const saveSnapshot = Effect.fn("DockerDatabaseStorage.saveSnapshot")( + (version: string, key: string) => + Effect.gen(function* () { + yield* selected; + const store = yield* getMarker; + if (!store.initialized) + return yield* errorFor("snapshot", "Database is not initialized"); + const readyPath = options.path.join( + options.instanceRoot, + ".supabase-database-ready.json", + ); + if ( + !(yield* options.fs + .exists(readyPath) + .pipe(Effect.mapError((cause) => errorFor("snapshot", cause)))) + ) + return yield* errorFor("snapshot", "Database is not ready"); + const ready = yield* options.fs + .readFileString(readyPath) + .pipe(Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(ReadyMarker)))); + if (ready.version !== version || ready.runtime !== options.runtime) + return yield* errorFor( + "snapshot", + "Database readiness marker does not match the requested configuration", + ); + const digest = yield* descriptorDigest(version, key); + const paths = snapshotPaths(store); + const root = paths.root; + const source = paths.source; + const target = `${root}/entries/${digest}`; + const token = yield* options.crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => errorFor("snapshot", cause)), + ); + const stage = `${root}/stages/${digest}-${token}`; + const descriptor = yield* encodeDescriptor({ + format: "supabase-database-snapshot-v1", + version, + runtime: options.runtime, + platform: process.platform, + arch: process.arch, + profile: "supabase", + keyDigest: digest, + }); + yield* runHelper( + `set -eu; mkdir -p ${root}; flock -x -w 120 ${shellQuote(`${root}/.lock`)} sh -eu -c ${shellQuote(`set -eu; mkdir -p ${root}/entries ${root}/stages; find ${root}/stages -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +; find ${root}/entries -mindepth 1 -maxdepth 1 -name '*.retired' -exec rm -rf -- {} +; trap 'rm -rf ${stage}' EXIT; if [ -e ${source}/postmaster.pid ]; then echo 'Cannot save a running database snapshot' >&2; exit 1; fi; if [ ! -f ${source}/PG_VERSION ]; then echo 'Cannot save snapshot: PG_VERSION is missing' >&2; exit 1; fi; actual=$(cat ${source}/PG_VERSION); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Cannot save snapshot: PostgreSQL major does not match requested version' >&2; exit 1; fi; bad=$(find ${source} \\( ! -type f ! -type d \\) -print -quit); if [ -n "$bad" ]; then echo "Cannot save snapshot: unsupported filesystem entry $bad" >&2; exit 1; fi; rm -rf ${stage}; mkdir -p ${stage}; cp -a --reflink=auto ${source} ${stage}/data; printf '%s' ${shellQuote(descriptor)} > ${stage}/descriptor.json; if [ -e ${target} ]; then rm -rf ${target}.retired; mv ${target} ${target}.retired; if ! mv ${stage} ${target}; then mv ${target}.retired ${target}; exit 1; fi; else mv ${stage} ${target}; fi; rm -rf ${target}.retired; touch ${target}; find ${root}/entries -mindepth 1 -maxdepth 1 -type d ! -name ${digest} ! -name '*.retired' -printf '%T@ %p\\n' | sort -rn | tail -n +3 | cut -d' ' -f2- | xargs -r rm -rf`)}; rm -rf ${shellQuote(stage)}`, + paths.mounts, + ); + }).pipe(Effect.mapError((cause) => errorFor("snapshot", cause))), + ); + const restoreSnapshot = Effect.fn("DockerDatabaseStorage.restoreSnapshot")( + (version: string, key: string) => + Effect.gen(function* () { + yield* selected; + const store = yield* getMarker; + const digest = yield* descriptorDigest(version, key); + const paths = snapshotPaths(store); + const root = paths.root; + const source = `${root}/entries/${digest}`; + const data = paths.source; + const token = yield* options.crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => errorFor("snapshot", cause)), + ); + const stage = + store.backend === "host" + ? `/instance/.supabase-restore-${digest}` + : `${root}/stages/restore-${digest}-${token}`; + const descriptor = yield* encodeDescriptor({ + format: "supabase-database-snapshot-v1", + version, + runtime: options.runtime, + platform: process.platform, + arch: process.arch, + profile: "supabase", + keyDigest: digest, + }); + const targetSetup = store.initialized ? `test -d ${data}` : `mkdir -p ${data}`; + const result = yield* runHelper( + `set -eu; mkdir -p ${root}; flock -x -w 120 ${shellQuote(`${root}/.lock`)} sh -eu -c ${shellQuote(`set -eu; mkdir -p ${root}/entries ${root}/stages; find ${root}/stages -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +; find ${root}/entries -mindepth 1 -maxdepth 1 -name '*.retired' -exec rm -rf -- {} +; if ! ${targetSetup}; then echo 'Initialized restore target data directory is missing' >&2; exit 1; fi; trap 'rm -rf ${stage}' EXIT; bad=$(find ${data} -mindepth 1 -print -quit); if [ -n "$bad" ]; then echo NONEMPTY; exit 0; fi; if [ ! -d ${source} ]; then echo MISS; exit 0; fi; if [ ! -f ${source}/descriptor.json ]; then echo 'Snapshot descriptor is missing' >&2; exit 1; fi; actual=$(cat ${source}/descriptor.json); expected=${shellQuote(descriptor)}; if [ "$actual" != "$expected" ]; then echo 'Snapshot descriptor does not match requested identity' >&2; exit 1; fi; bad=$(find ${source}/data \\( ! -type f ! -type d \\) -print -quit); if [ -n "$bad" ]; then echo "Snapshot contains unsupported filesystem entry $bad" >&2; exit 1; fi; if [ -e ${source}/data/postmaster.pid ]; then echo 'Snapshot contains postmaster.pid' >&2; exit 1; fi; rm -rf ${stage}; cp -a --reflink=auto ${source}/data ${stage}; actual=$(cat ${stage}/PG_VERSION); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Snapshot PostgreSQL major does not match requested version' >&2; exit 1; fi; rmdir ${data}; mv ${stage} ${data}; touch ${source}; echo HIT`)}; rm -rf ${shellQuote(stage)}`, + paths.mounts, + ); + if (result === "MISS") return false; + if (result === "NONEMPTY") + return yield* errorFor("restore", "Restore target data directory must be empty"); + if (result !== "HIT") + return yield* errorFor("restore", "Docker snapshot restore did not publish"); + yield* publishReadyMarker(version); + yield* writeMarker({ ...store, initialized: true }); + return true; + }).pipe(Effect.mapError((cause) => errorFor("snapshot", cause))), + ); + return { + prepare: setup, + needsDataChown, + mount, + markInitialized, + removeData, + destroyData, + saveSnapshot, + restoreSnapshot, + }; + }), +); diff --git a/packages/stack/tests/whole-stack/workloads.ts b/packages/stack/tests/whole-stack/workloads.ts index 2193493cfa..1e496eab17 100644 --- a/packages/stack/tests/whole-stack/workloads.ts +++ b/packages/stack/tests/whole-stack/workloads.ts @@ -5,6 +5,7 @@ import { service, type Runtime, type WholeStack } from "./fixture.ts"; export const WorkloadSnapshot = Schema.Struct({ identities: Schema.Array(Schema.String), + managedIdentities: Schema.Array(Schema.String), markers: Schema.Array(Schema.String), }); export type WorkloadSnapshot = Schema.Schema.Type; @@ -29,7 +30,11 @@ const snapshotOutput = (runtime: Runtime, fixture: WholeStack, includeStopped: b .split("\n") .map((line) => line.trim()) .filter((line) => line.includes(stackMarker) || line.includes(databaseMarker)); - return { identities, markers: [stackMarker, databaseMarker] } satisfies WorkloadSnapshot; + return { + identities, + managedIdentities: [], + markers: [stackMarker, databaseMarker], + } satisfies WorkloadSnapshot; } const marker = `com.supabase.stack=${fixture.stack.id}`; const output = yield* commandOutput("docker", [ @@ -38,13 +43,23 @@ const snapshotOutput = (runtime: Runtime, fixture: WholeStack, includeStopped: b "--filter", `label=${marker}`, "--format", - '{{.ID}} {{.Label "com.supabase.instance"}}', + '{{.ID}}\t{{.Label "com.supabase.instance"}}\t{{.Label "com.supabase.stack-managed"}}', ]); + const records = output + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => { + const [id, instance, managed] = line.split("\t"); + return { identity: `${id ?? ""} ${instance ?? ""}`.trim(), managed }; + }); return { - identities: output - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0), + identities: records + .filter((record) => record.managed !== "true") + .map((record) => record.identity), + managedIdentities: records + .filter((record) => record.managed === "true") + .map((record) => record.identity), markers: [marker], } satisfies WorkloadSnapshot; }); @@ -60,6 +75,7 @@ export const assertWorkloadsGone = Effect.fn("WholeStack.assertWorkloadsGone")( expect(snapshot.identities.length).toBeGreaterThan(0); const current = yield* snapshotOutput(runtime, fixture, includeStopped); if (runtime === "docker") expect(current.identities).toEqual([]); + if (runtime === "docker" && includeStopped) expect(current.managedIdentities).toEqual([]); for (const marker of snapshot.markers) expect(current.identities.some((identity) => identity.includes(marker))).toBe(false); for (const identity of snapshot.identities) From fd9b2f4a3e11243ef216a02ca7c8c54583725a9d Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 22 Sep 2026 09:39:39 +0200 Subject: [PATCH 02/20] fix(stack): tighten snapshot ownership and cleanup --- .../db/diff/diff.stack-cache.e2e.test.ts | 73 +++++++++- ...elta-next-shadow.stack.integration.test.ts | 8 ++ packages/stack/src/storage/DirectoryCopy.ts | 2 +- .../DockerDatabaseStorage.integration.test.ts | 136 +++++++++++++++--- .../src/storage/DockerDatabaseStorage.ts | 85 ++++++++--- 5 files changed, 255 insertions(+), 49 deletions(-) diff --git a/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts b/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts index 79869c0956..1ab1ce5ed9 100644 --- a/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts +++ b/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts @@ -1,7 +1,8 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Effect, FileSystem, Path, Redacted } from "effect"; +import { Data, Effect, Exit, FileSystem, Path, Predicate, Redacted, Schema, Stream } from "effect"; import { FetchHttpClient } from "effect/unstable/http"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { create as createStack } from "@supabase/stack/effect"; import { tmpdir } from "node:os"; import { runSupabaseEffect } from "../../../../tests/helpers/cli.ts"; @@ -30,6 +31,38 @@ enabled = false enabled = false `; +const storageMarker = Schema.Struct({ + backend: Schema.Literals(["docker", "host"]), + volume: Schema.optionalKey(Schema.String), +}); + +class DockerCleanupError extends Data.TaggedError("DockerCleanupError")<{ + readonly message: string; +}> {} + +const removeDockerVolume = Effect.fn("DbDiffStackCacheE2e.removeDockerVolume")((volume: string) => + Effect.scoped( + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + ChildProcess.make("docker", ["volume", "rm", volume], { stdin: "ignore" }), + ); + const [stderr, code] = yield* Effect.all( + [ + child.stdout.pipe(Stream.runDrain), + child.stderr.pipe(Stream.decodeText, Stream.mkString), + child.exitCode, + ], + { concurrency: "unbounded" }, + ).pipe(Effect.map(([, stderr, code]) => [stderr, code] as const)); + if (Number(code) !== 0 && !/no such volume/iu.test(stderr)) + return yield* new DockerCleanupError({ + message: `docker volume rm ${volume} failed: ${stderr.trim() || `exit ${code}`}`, + }); + }), + ), +); + const composeStack = Effect.fn("DbDiffStackCacheE2e.composeStack")(function* ( root: string, home: string, @@ -41,12 +74,40 @@ const composeStack = Effect.fn("DbDiffStackCacheE2e.composeStack")(function* ( cacheRoot: `${home}/cache/stack`, runtime, }); + let databaseId: string | undefined; yield* Effect.addFinalizer(() => - stack.destroy.pipe( - Effect.catch((cause) => Effect.die(new Error(`stack cleanup failed: ${cause.message}`))), - ), + Effect.gen(function* () { + const marker = yield* Effect.gen(function* () { + if (runtime !== "docker" || databaseId === undefined) return undefined; + const fs = yield* FileSystem.FileSystem; + const markerText = yield* fs + .readFileString( + `${home}/stacks/${stack.id}/data/${databaseId}/.supabase-database-storage.json`, + ) + .pipe( + Effect.catchIf( + (cause) => Predicate.isTagged(cause.reason, "NotFound"), + () => Effect.succeed(undefined), + ), + ); + if (markerText === undefined) return undefined; + return yield* Schema.decodeEffect(Schema.fromJsonString(storageMarker))(markerText); + }).pipe(Effect.exit); + const destroyed = yield* stack.destroy.pipe(Effect.exit); + const volume = + Exit.isSuccess(marker) && marker.value !== undefined && marker.value.backend === "docker" + ? marker.value.volume + : undefined; + const removed = + volume === undefined + ? Exit.succeed(undefined) + : yield* removeDockerVolume(volume).pipe(Effect.exit); + if (Exit.isFailure(destroyed)) return yield* Effect.failCause(destroyed.cause); + if (Exit.isFailure(marker)) return yield* Effect.failCause(marker.cause); + if (Exit.isFailure(removed)) return yield* Effect.failCause(removed.cause); + }).pipe(Effect.catchCause((cause) => Effect.die(cause))), ); - yield* stack.composition.supabase([ + const [database] = yield* stack.composition.supabase([ { service: "database", config: { @@ -58,6 +119,8 @@ const composeStack = Effect.fn("DbDiffStackCacheE2e.composeStack")(function* ( endpoints: { sql: { port: "auto" } }, }, ]); + if (database === undefined) return yield* Effect.die("database composition missing"); + databaseId = database.id; yield* stack.composition.start; return stack; }); diff --git a/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts b/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts index f7c433462d..c2250eb404 100644 --- a/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts +++ b/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts @@ -78,6 +78,14 @@ describe("pg-delta next stack shadow provisioning", () => { "keeps migrations on the migration shadow and destroys both shadows with the caller scope", () => Effect.gen(function* () { + const previousShadowCache = process.env["SUPABASE_SHADOW_CACHE"]; + process.env["SUPABASE_SHADOW_CACHE"] = "1"; + yield* Effect.addFinalizer(() => + Effect.sync(() => { + if (previousShadowCache === undefined) delete process.env["SUPABASE_SHADOW_CACHE"]; + else process.env["SUPABASE_SHADOW_CACHE"] = previousShadowCache; + }), + ); const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "pgdelta-next-stack-" }); yield* fs.makeDirectory(`${root}/supabase/migrations`, { recursive: true }); diff --git a/packages/stack/src/storage/DirectoryCopy.ts b/packages/stack/src/storage/DirectoryCopy.ts index 4af4d13253..c6f3746f57 100644 --- a/packages/stack/src/storage/DirectoryCopy.ts +++ b/packages/stack/src/storage/DirectoryCopy.ts @@ -138,7 +138,7 @@ const copyTree = ( for (const entry of entries) { const childSource = path.join(source, entry); const childDestination = path.join(destination, entry); - const childStats = yield* inspect(childSource, destination); + const childStats = yield* inspect(childSource, childDestination); if (childStats.isSymbolicLink()) return yield* errorFor("validate", childSource, childDestination, "symbolic link"); if (childStats.isDirectory()) { diff --git a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts index 0a552d86d6..697f150baa 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts @@ -18,7 +18,7 @@ import { makeDatabaseSnapshots } from "../services/DatabaseSnapshot.ts"; import { makeDockerDatabaseStorage } from "./DockerDatabaseStorage.ts"; const helperImage = - "debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; + "public.ecr.aws/docker/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; const Marker = Schema.Struct({ backend: Schema.Literals(["docker", "host"]), volume: Schema.optionalKey(Schema.String), @@ -58,7 +58,9 @@ const docker = Effect.fn("DockerStorageTest.docker")((args: ReadonlyArray { yield* fs.makeDirectory(cacheRoot, { recursive: true }); const container = yield* makeContainerRuntime({ engine: "docker" }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const makeStorage = (instanceId: string, instanceRoot: string) => + const makeStorageAt = ( + instanceId: string, + instanceRoot: string, + storageRootValue: string, + cacheRootValue: string, + ) => makeDockerDatabaseStorage({ runtime: "docker", stackId: "storage-test", instanceId, instanceRoot, - root: storageRoot, - cacheRoot, + root: storageRootValue, + cacheRoot: cacheRootValue, fs, path, crypto, container, spawner, }); + const makeStorage = (instanceId: string, instanceRoot: string) => + makeStorageAt(instanceId, instanceRoot, storageRoot, cacheRoot); const source = yield* makeStorage("source", sourceRoot); const target = yield* makeStorage("target", targetRoot); const ownerScope = yield* Scope.Scope; @@ -144,6 +153,38 @@ describe("Docker database storage", { timeout: 120_000 }, () => { ); yield* source.saveSnapshot("17", "roundtrip"); + const copiedRoot = path.join(root, "copied-state", "stack", "data", "source"); + const copiedStorageRoot = path.join(root, "copied-state", "stack", "data"); + const copiedCacheRoot = path.join(root, "copied-cache"); + yield* fs.makeDirectory(copiedRoot, { recursive: true }); + yield* fs.makeDirectory(copiedCacheRoot, { recursive: true }); + yield* fs.writeFileString( + path.join(copiedRoot, ".supabase-database-storage.json"), + yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), + ); + const copied = yield* makeStorageAt( + "source", + copiedRoot, + copiedStorageRoot, + copiedCacheRoot, + ); + const mountError = yield* copied.mount("17").pipe(Effect.flip); + expect(mountError.message).toContain("another state directory"); + const removeError = yield* copied.removeData("17").pipe(Effect.flip); + expect(removeError.message).toContain("another state directory"); + const destroyError = yield* copied.destroyData("17").pipe(Effect.flip); + expect(destroyError.message).toContain("another state directory"); + yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `test "$(cat /store/${sourceMarker.namespace}/data/fixture)" = source`, + ]); + yield* target.prepare("17"); expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(true); const targetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( @@ -412,12 +453,18 @@ describe("Docker database storage", { timeout: 120_000 }, () => { }), ); if (volume === undefined) return yield* new DockerTestError({ message: "Missing volume" }); - yield* docker(["volume", "rm", volume]); + const removedVolume = volume; + yield* docker(["volume", "rm", removedVolume]); yield* Effect.scoped( Effect.gen(function* () { const storage = yield* makeStorage(); const prepare = yield* storage.prepare("17").pipe(Effect.exit); expect(prepare).toSatisfy((exit) => Exit.isFailure(exit)); + const removeError = yield* storage.removeData("17").pipe(Effect.flip); + expect(removeError.message).toMatch(/no such volume/iu); + expect(yield* docker(["volume", "inspect", removedVolume]).pipe(Effect.exit)).toSatisfy( + (exit) => Exit.isFailure(exit), + ); expect(yield* storage.destroyData("17").pipe(Effect.exit)).toSatisfy((exit) => Exit.isSuccess(exit), ); @@ -472,12 +519,31 @@ describe("Docker database storage", { timeout: 120_000 }, () => { container, spawner, }); + const ownerScope = yield* Scope.Scope; + yield* Scope.addFinalizer( + ownerScope, + storage.destroyData("17").pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.catchCause((cause) => Effect.die(cause)), + ), + ); yield* storage.prepare("17"); const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( yield* fs.readFileString(path.join(instanceRoot, ".supabase-database-storage.json")), ); expect(marker.backend).toBe("host"); expect(marker.initialized).toBe(true); + const expectedOwnership = yield* docker([ + "run", + "--rm", + "--mount", + `type=bind,src=${instanceRoot},dst=/instance`, + helperImage, + "/bin/sh", + "-c", + "stat -c '%u:%g' /instance/data/PG_VERSION", + ]); + if (process.platform === "linux") expect(expectedOwnership).toBe("100:101"); yield* storage.saveSnapshot("17", "adopted"); const nativeRoot = path.join(root, "native"); yield* fs.makeDirectory(path.join(nativeRoot, "data"), { recursive: true }); @@ -499,23 +565,25 @@ describe("Docker database storage", { timeout: 120_000 }, () => { expect(yield* nativeSnapshots.restoreSnapshot("native")).toBe(true); yield* storage.removeData("17"); expect(yield* storage.restoreSnapshot("17", "adopted")).toBe(true); + expect( + yield* docker([ + "run", + "--rm", + "--mount", + `type=bind,src=${instanceRoot},dst=/instance`, + helperImage, + "/bin/sh", + "-c", + "stat -c '%u:%g' /instance/data/PG_VERSION", + ]), + ).toBe(expectedOwnership); yield* storage.removeData("17"); yield* storage.destroyData("17"); expect(yield* fs.exists(path.join(instanceRoot, ".supabase-database-storage.json"))).toBe( true, ); expect(yield* fs.exists(dataRoot)).toBe(false); - yield* docker([ - "run", - "--rm", - "--mount", - `type=bind,src=${cacheRoot},dst=/cache`, - helperImage, - "/bin/sh", - "-c", - "rm -rf /cache/*", - ]); - yield* fs.remove(cacheRoot, { recursive: true, force: true }); + yield* fs.remove(cacheRoot, { recursive: true }); }), ).pipe(Effect.provide(NodeServices.layer)), ); @@ -537,21 +605,38 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* fs.makeDirectory(cacheRoot, { recursive: true }); const container = yield* makeContainerRuntime({ engine: "docker" }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const makeStorage = (instanceId: string, instanceRoot: string) => + const makeStorageWithCache = ( + instanceId: string, + instanceRoot: string, + cacheRootValue: string, + ) => makeDockerDatabaseStorage({ runtime: "docker", stackId: "storage-reopen-test", instanceId, instanceRoot, root: storageRoot, - cacheRoot, + cacheRoot: cacheRootValue, fs, path, crypto, container, spawner, }); + const makeStorage = (instanceId: string, instanceRoot: string) => + makeStorageWithCache(instanceId, instanceRoot, cacheRoot); let sourceVolume: string | undefined; + const ownerScope = yield* Scope.Scope; + yield* Scope.addFinalizer( + ownerScope, + Effect.gen(function* () { + if (sourceVolume !== undefined) yield* docker(["volume", "rm", sourceVolume]); + yield* fs.remove(cacheRoot, { recursive: true, force: true }); + }).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.catchCause((cause) => Effect.die(cause)), + ), + ); yield* Effect.scoped( Effect.gen(function* () { const source = yield* makeStorage("source", sourceRoot); @@ -585,9 +670,18 @@ describe("Docker database storage", { timeout: 120_000 }, () => { ); yield* Effect.scoped( Effect.gen(function* () { + const changedCacheRoot = path.join(root, "changed-cache"); + yield* fs.makeDirectory(changedCacheRoot, { recursive: true }); + const changedCache = yield* makeStorageWithCache( + "source", + sourceRoot, + changedCacheRoot, + ); + yield* changedCache.prepare("17"); + yield* changedCache.removeData("17"); + expect(yield* changedCache.restoreSnapshot("17", "reopened")).toBe(false); const source = yield* makeStorage("source", sourceRoot); yield* source.prepare("17"); - yield* source.saveSnapshot("17", "reopened"); yield* source.removeData("17"); expect(yield* source.restoreSnapshot("17", "reopened")).toBe(true); yield* source.saveSnapshot("17", "reopened-again"); @@ -611,8 +705,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* target.destroyData("17"); }), ); - if (sourceVolume !== undefined) yield* docker(["volume", "rm", sourceVolume]); - yield* fs.remove(cacheRoot, { recursive: true, force: true }); }), ).pipe(Effect.provide(NodeServices.layer)), ); diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index cae4ab01b0..9c99261c92 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -17,7 +17,7 @@ import type { ContainerRuntime } from "../runtime/Container.ts"; import type { DatabaseRuntime } from "../services/Database.ts"; const HELPER_IMAGE = - "docker.io/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; + "public.ecr.aws/docker/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; const Marker = Schema.Struct({ backend: Schema.Literals(["docker", "host"]), @@ -159,18 +159,28 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") const present = yield* options.fs .exists(markerPath) .pipe(Effect.mapError((cause) => errorFor("marker", cause))); - if (present) - return yield* options.fs.readFileString(markerPath).pipe( - Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(Marker))), - Effect.flatMap(validateMarker), - Effect.mapError((cause) => errorFor("marker", cause)), - ); yield* options.fs .makeDirectory(options.cacheRoot, { recursive: true }) .pipe(Effect.mapError((cause) => errorFor("identity", cause))); const cacheRoot = yield* options.fs .realPath(options.cacheRoot) .pipe(Effect.mapError((cause) => errorFor("identity", cause))); + const cacheNamespace = `cache-${(yield* hash(cacheRoot)).slice(0, 32)}`; + if (present) { + const marker = yield* options.fs.readFileString(markerPath).pipe( + Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(Marker))), + Effect.flatMap(validateMarker), + Effect.mapError((cause) => errorFor("marker", cause)), + ); + if (marker.cacheNamespace !== cacheNamespace) { + const updated = { ...marker, cacheNamespace }; + yield* options.fs + .writeFileString(markerPath, yield* encodeMarker(updated), { mode: 0o600 }) + .pipe(Effect.mapError((cause) => errorFor("marker", cause))); + return updated; + } + return marker; + } const hostData = options.path.join(options.instanceRoot, "data"); const initialized = (yield* options.fs @@ -183,7 +193,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") const value: Marker = { backend: "host", namespace: dataNamespace, - cacheNamespace: `cache-${(yield* hash(cacheRoot)).slice(0, 32)}`, + cacheNamespace, initialized, }; const encoded = yield* encodeMarker(value); @@ -236,7 +246,11 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ); if (validMarker.volume === undefined) return yield* errorFor("data", "Recorded Docker storage volume is missing"); - const volume = validMarker.volume; + if (validMarker.volume !== volume) + return yield* errorFor( + "data", + "Recorded Docker database storage belongs to another state directory", + ); yield* engineCommand(["volume", "inspect", volume]).pipe( Effect.catchTag("DockerDatabaseStorageError", (cause) => !validMarker.initialized && /(?:no such volume|not found)/iu.test(cause.message) @@ -252,9 +266,16 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") : Effect.fail(cause), ), ); - return validMarker; } - return validMarker; + const updated = + validMarker.cacheNamespace === `cache-${cacheDigest.slice(0, 32)}` + ? validMarker + : { ...validMarker, cacheNamespace: `cache-${cacheDigest.slice(0, 32)}` }; + if (updated !== validMarker) + yield* options.fs + .writeFileString(markerPath, yield* encodeMarker(updated), { mode: 0o600 }) + .pipe(Effect.mapError((cause) => errorFor("marker", cause))); + return updated; } const hostData = options.path.join(options.instanceRoot, "data"); if ( @@ -346,6 +367,27 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") }), ).pipe(Effect.mapError((cause) => errorFor("engine", cause))), ); + const validateDockerMarkerIdentity = (marker: Marker) => + Effect.gen(function* () { + const daemonId = yield* engineCommand(["info", "--format", "{{.ID}}"]).pipe( + Effect.map((value) => value.trim()), + ); + if (marker.daemonId !== daemonId) + return yield* errorFor( + "destroy", + "Recorded Docker database storage belongs to another daemon; switch back to the original Docker context before destroying it", + ); + const canonicalStateRoot = yield* options.fs + .realPath(stateRoot) + .pipe(Effect.mapError((cause) => errorFor("identity", cause))); + const stateDigest = yield* hash(`${canonicalStateRoot}\0${daemonId}`); + const expectedVolume = `supabase-db-${stateDigest.slice(0, 32)}`; + if (marker.volume !== expectedVolume) + return yield* errorFor( + "destroy", + "Recorded Docker database storage belongs to another state directory", + ); + }); const helperId = yield* Ref.make(undefined); const helperCleanupPending = yield* Ref.make(false); @@ -652,14 +694,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ); yield* removeHelper(); } else { - const daemonId = yield* engineCommand(["info", "--format", "{{.ID}}"]).pipe( - Effect.map((value) => value.trim()), - ); - if (marker.daemonId !== daemonId) - return yield* errorFor( - "destroy", - "Recorded Docker database storage belongs to another daemon; switch back to the original Docker context before destroying it", - ); + yield* validateDockerMarkerIdentity(marker); if (marker.volume === undefined) return yield* errorFor("destroy", "Recorded Docker storage volume is missing"); const volumeExists = yield* engineCommand(["volume", "inspect", marker.volume]).pipe( @@ -754,8 +789,12 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") profile: "supabase", keyDigest: digest, }); + const cacheOwnership = + store.backend === "host" + ? `; owner=$(stat -c "%u:%g" /cache); mkdir -p /cache/stack-database-snapshots-helper; chown "$owner" /cache/stack-database-snapshots-helper; chown -R "$owner" ${root}` + : ""; yield* runHelper( - `set -eu; mkdir -p ${root}; flock -x -w 120 ${shellQuote(`${root}/.lock`)} sh -eu -c ${shellQuote(`set -eu; mkdir -p ${root}/entries ${root}/stages; find ${root}/stages -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +; find ${root}/entries -mindepth 1 -maxdepth 1 -name '*.retired' -exec rm -rf -- {} +; trap 'rm -rf ${stage}' EXIT; if [ -e ${source}/postmaster.pid ]; then echo 'Cannot save a running database snapshot' >&2; exit 1; fi; if [ ! -f ${source}/PG_VERSION ]; then echo 'Cannot save snapshot: PG_VERSION is missing' >&2; exit 1; fi; actual=$(cat ${source}/PG_VERSION); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Cannot save snapshot: PostgreSQL major does not match requested version' >&2; exit 1; fi; bad=$(find ${source} \\( ! -type f ! -type d \\) -print -quit); if [ -n "$bad" ]; then echo "Cannot save snapshot: unsupported filesystem entry $bad" >&2; exit 1; fi; rm -rf ${stage}; mkdir -p ${stage}; cp -a --reflink=auto ${source} ${stage}/data; printf '%s' ${shellQuote(descriptor)} > ${stage}/descriptor.json; if [ -e ${target} ]; then rm -rf ${target}.retired; mv ${target} ${target}.retired; if ! mv ${stage} ${target}; then mv ${target}.retired ${target}; exit 1; fi; else mv ${stage} ${target}; fi; rm -rf ${target}.retired; touch ${target}; find ${root}/entries -mindepth 1 -maxdepth 1 -type d ! -name ${digest} ! -name '*.retired' -printf '%T@ %p\\n' | sort -rn | tail -n +3 | cut -d' ' -f2- | xargs -r rm -rf`)}; rm -rf ${shellQuote(stage)}`, + `set -eu; mkdir -p ${root}; flock -x -w 120 ${shellQuote(`${root}/.lock`)} sh -eu -c ${shellQuote(`set -eu; mkdir -p ${root}/entries ${root}/stages; find ${root}/stages -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +; find ${root}/entries -mindepth 1 -maxdepth 1 -name '*.retired' -exec rm -rf -- {} +; trap 'rm -rf ${stage}${cacheOwnership}' EXIT; if [ -e ${source}/postmaster.pid ]; then echo 'Cannot save a running database snapshot' >&2; exit 1; fi; if [ ! -f ${source}/PG_VERSION ]; then echo 'Cannot save snapshot: PG_VERSION is missing' >&2; exit 1; fi; actual=$(cat ${source}/PG_VERSION); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Cannot save snapshot: PostgreSQL major does not match requested version' >&2; exit 1; fi; bad=$(find ${source} \\( ! -type f ! -type d \\) -print -quit); if [ -n "$bad" ]; then echo "Cannot save snapshot: unsupported filesystem entry $bad" >&2; exit 1; fi; rm -rf ${stage}; mkdir -p ${stage}; cp -a --reflink=auto ${source} ${stage}/data; printf '%s' ${shellQuote(descriptor)} > ${stage}/descriptor.json; if [ -e ${target} ]; then rm -rf ${target}.retired; mv ${target} ${target}.retired; if ! mv ${stage} ${target}; then mv ${target}.retired ${target}; exit 1; fi; else mv ${stage} ${target}; fi; rm -rf ${target}.retired; touch ${target}; find ${root}/entries -mindepth 1 -maxdepth 1 -type d ! -name ${digest} ! -name '*.retired' -printf '%T@ %p\\n' | sort -rn | tail -n +3 | cut -d' ' -f2- | xargs -r rm -rf`)}; rm -rf ${shellQuote(stage)}`, paths.mounts, ); }).pipe(Effect.mapError((cause) => errorFor("snapshot", cause))), @@ -787,8 +826,12 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") keyDigest: digest, }); const targetSetup = store.initialized ? `test -d ${data}` : `mkdir -p ${data}`; + const cacheOwnership = + store.backend === "host" + ? `; owner=$(stat -c "%u:%g" /cache); mkdir -p /cache/stack-database-snapshots-helper; chown "$owner" /cache/stack-database-snapshots-helper; chown -R "$owner" ${root}` + : ""; const result = yield* runHelper( - `set -eu; mkdir -p ${root}; flock -x -w 120 ${shellQuote(`${root}/.lock`)} sh -eu -c ${shellQuote(`set -eu; mkdir -p ${root}/entries ${root}/stages; find ${root}/stages -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +; find ${root}/entries -mindepth 1 -maxdepth 1 -name '*.retired' -exec rm -rf -- {} +; if ! ${targetSetup}; then echo 'Initialized restore target data directory is missing' >&2; exit 1; fi; trap 'rm -rf ${stage}' EXIT; bad=$(find ${data} -mindepth 1 -print -quit); if [ -n "$bad" ]; then echo NONEMPTY; exit 0; fi; if [ ! -d ${source} ]; then echo MISS; exit 0; fi; if [ ! -f ${source}/descriptor.json ]; then echo 'Snapshot descriptor is missing' >&2; exit 1; fi; actual=$(cat ${source}/descriptor.json); expected=${shellQuote(descriptor)}; if [ "$actual" != "$expected" ]; then echo 'Snapshot descriptor does not match requested identity' >&2; exit 1; fi; bad=$(find ${source}/data \\( ! -type f ! -type d \\) -print -quit); if [ -n "$bad" ]; then echo "Snapshot contains unsupported filesystem entry $bad" >&2; exit 1; fi; if [ -e ${source}/data/postmaster.pid ]; then echo 'Snapshot contains postmaster.pid' >&2; exit 1; fi; rm -rf ${stage}; cp -a --reflink=auto ${source}/data ${stage}; actual=$(cat ${stage}/PG_VERSION); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Snapshot PostgreSQL major does not match requested version' >&2; exit 1; fi; rmdir ${data}; mv ${stage} ${data}; touch ${source}; echo HIT`)}; rm -rf ${shellQuote(stage)}`, + `set -eu; mkdir -p ${root}; flock -x -w 120 ${shellQuote(`${root}/.lock`)} sh -eu -c ${shellQuote(`set -eu; mkdir -p ${root}/entries ${root}/stages; find ${root}/stages -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +; find ${root}/entries -mindepth 1 -maxdepth 1 -name '*.retired' -exec rm -rf -- {} +; trap 'rm -rf ${stage}${cacheOwnership}' EXIT; if ! ${targetSetup}; then echo 'Initialized restore target data directory is missing' >&2; exit 1; fi; bad=$(find ${data} -mindepth 1 -print -quit); if [ -n "$bad" ]; then echo NONEMPTY; exit 0; fi; if [ ! -d ${source} ]; then echo MISS; exit 0; fi; if [ ! -f ${source}/descriptor.json ]; then echo 'Snapshot descriptor is missing' >&2; exit 1; fi; actual=$(cat ${source}/descriptor.json); expected=${shellQuote(descriptor)}; if [ "$actual" != "$expected" ]; then echo 'Snapshot descriptor does not match requested identity' >&2; exit 1; fi; bad=$(find ${source}/data \\( ! -type f ! -type d \\) -print -quit); if [ -n "$bad" ]; then echo "Snapshot contains unsupported filesystem entry $bad" >&2; exit 1; fi; if [ -e ${source}/data/postmaster.pid ]; then echo 'Snapshot contains postmaster.pid' >&2; exit 1; fi; rm -rf ${stage}; cp -a --reflink=auto ${source}/data ${stage}; actual=$(cat ${stage}/PG_VERSION); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Snapshot PostgreSQL major does not match requested version' >&2; exit 1; fi; ${store.backend === "host" ? `chown -R 100:101 ${stage};` : ""} rmdir ${data}; mv ${stage} ${data}; touch ${source}; echo HIT`)}; rm -rf ${shellQuote(stage)}`, paths.mounts, ); if (result === "MISS") return false; From 31820ab475e18b86b2f4eef746d8c980b4dbafc9 Mon Sep 17 00:00:00 2001 From: Andrew Valleteau Date: Wed, 23 Sep 2026 10:11:48 +0200 Subject: [PATCH 03/20] perf(cli): compile the CLI with bytecode (#6708) Compiled CLI builds now enable minification, ESM, and bytecode at depth 2 to reduce startup work. Release, local, and E2E binaries share the same compilation options, including the local npm release workflow. Depth 2 limits the binary-size cost compared with compiling all nested functions ahead of time. --------- Co-authored-by: Julien Goux --- apps/cli/scripts/build-binary.ts | 2 ++ apps/cli/scripts/build.ts | 4 ++-- apps/cli/scripts/bundle-externals.ts | 9 --------- apps/cli/scripts/compile-options.ts | 10 ++++++++++ ...011-cli-release-and-distribution-strategy.md | 2 +- tools/release/local-release.ts | 17 +++++++++++++++-- 6 files changed, 30 insertions(+), 14 deletions(-) create mode 100644 apps/cli/scripts/compile-options.ts diff --git a/apps/cli/scripts/build-binary.ts b/apps/cli/scripts/build-binary.ts index ceda84411a..c5be638e5a 100644 --- a/apps/cli/scripts/build-binary.ts +++ b/apps/cli/scripts/build-binary.ts @@ -2,6 +2,7 @@ import { bundleServeMainTemplate } from "../src/shared/functions/serve-main-bund import { bundleStackFunctionsServeMainTemplate } from "../src/command-internal/stack-functions-bundler.ts"; import { Effect } from "effect"; import { OXFMT_OPTIONAL_PLUGIN_EXTERNALS } from "./bundle-externals.ts"; +import { compileOptions } from "./compile-options.ts"; /** * Compiles the CLI to a standalone binary, run via `pnpm build:binary`. Embeds the pre-bundled @@ -21,6 +22,7 @@ if (packageJson.version === undefined || packageJson.version.length === 0) { const result = await Bun.build({ entrypoints: [entrypoint], compile: { outfile }, + ...compileOptions, external: [...OXFMT_OPTIONAL_PLUGIN_EXTERNALS], define: { SUPABASE_CLI_VERSION: JSON.stringify(packageJson.version), diff --git a/apps/cli/scripts/build.ts b/apps/cli/scripts/build.ts index 48b3441adb..4412e5d96a 100644 --- a/apps/cli/scripts/build.ts +++ b/apps/cli/scripts/build.ts @@ -8,6 +8,7 @@ import { Effect } from "effect"; import { bundleServeMainTemplate } from "../src/shared/functions/serve-main-bundler.ts"; import { bundleStackFunctionsServeMainTemplate } from "../src/command-internal/stack-functions-bundler.ts"; import { OXFMT_OPTIONAL_PLUGIN_EXTERNALS } from "./bundle-externals.ts"; +import { compileOptions } from "./compile-options.ts"; import { darwinBinaries, MACOS_IDENTIFIERS } from "./macos-signing.ts"; const MUSL_TARGETS = [ @@ -121,6 +122,7 @@ function libcForBunTarget(target: string): "glibc" | "musl" | "" { async function runBunBuild(config: Bun.BuildConfig) { const result = await Bun.build({ ...config, + ...compileOptions, external: [...(config.external ?? []), ...OXFMT_OPTIONAL_PLUGIN_EXTERNALS], }); for (const log of result.logs) { @@ -139,7 +141,6 @@ async function buildTarget(target: (typeof TARGETS)[number]) { await runBunBuild({ entrypoints: [entrypoint], compile: { target: target.bunTarget, outfile }, - minify: true, define: { ...buildDefines, SUPABASE_CLI_VERSION: JSON.stringify(version), @@ -278,7 +279,6 @@ async function buildMuslBinaries() { await runBunBuild({ entrypoints: [entrypoint], compile: { target: target.bunTarget, outfile }, - minify: true, define: { ...buildDefines, SUPABASE_CLI_VERSION: JSON.stringify(version), diff --git a/apps/cli/scripts/bundle-externals.ts b/apps/cli/scripts/bundle-externals.ts index 25d7760b11..5cee5d7355 100644 --- a/apps/cli/scripts/bundle-externals.ts +++ b/apps/cli/scripts/bundle-externals.ts @@ -13,12 +13,3 @@ export const OXFMT_OPTIONAL_PLUGIN_EXTERNALS = [ "prettier-plugin-astro", "prettier-plugin-marko", ] as const; - -/** - * {@link OXFMT_OPTIONAL_PLUGIN_EXTERNALS} as `--external=` CLI arguments, for `bun build` - * invocations that shell out (e.g. `tools/release/local-release.ts`) rather than calling the - * `Bun.build()` object API. - */ -export const oxfmtExternalArgs = OXFMT_OPTIONAL_PLUGIN_EXTERNALS.map( - (name) => `--external=${name}`, -); diff --git a/apps/cli/scripts/compile-options.ts b/apps/cli/scripts/compile-options.ts new file mode 100644 index 0000000000..19392766ac --- /dev/null +++ b/apps/cli/scripts/compile-options.ts @@ -0,0 +1,10 @@ +/** + * Shared compile options keep release, local, and E2E builds exercising the shipped binary + * compilation behavior. `bytecodeDepth` is accepted by Bun 1.4.1 but is not yet in bun-types. + */ +export const compileOptions = { + minify: true, + bytecode: true, + bytecodeDepth: 2, + format: "esm" as const, +}; diff --git a/docs/adr/0011-cli-release-and-distribution-strategy.md b/docs/adr/0011-cli-release-and-distribution-strategy.md index 613186eff3..d94538dd61 100644 --- a/docs/adr/0011-cli-release-and-distribution-strategy.md +++ b/docs/adr/0011-cli-release-and-distribution-strategy.md @@ -45,7 +45,7 @@ Most of the above is **already implemented and validated** in this repo (build p `bun build --compile` produces a single static binary per target triple (see `[apps/cli/scripts/build.ts](../../apps/cli/scripts/build.ts)` lines 94–103): ```ts -await $`bun build ${entrypoint} --compile --minify --target=${target.bunTarget} --outfile=${outfile}`; +await $`bun build ${entrypoint} --compile --minify --bytecode --bytecode-depth=2 --format=esm --target=${target.bunTarget} --outfile=${outfile}`; ``` This matches the artifact shape of the current Go releases exactly: one `supabase` (or `supabase.exe`) per `{darwin,linux,windows} × {arm64,x64}` plus `linux-{arm64,x64}-musl`. Every downstream channel — Homebrew, Scoop, deb/rpm/apk, `.tar.gz`/`.zip` on GitHub Releases — receives the same drop-in binary the Go CLI used to produce. No installer logic changes downstream. diff --git a/tools/release/local-release.ts b/tools/release/local-release.ts index c2ae5891cf..a3b1367911 100644 --- a/tools/release/local-release.ts +++ b/tools/release/local-release.ts @@ -16,7 +16,8 @@ import { tmpdir } from "node:os"; import path from "node:path"; import process from "node:process"; import { parseArgs } from "node:util"; -import { oxfmtExternalArgs } from "../../apps/cli/scripts/bundle-externals.ts"; +import { OXFMT_OPTIONAL_PLUGIN_EXTERNALS } from "../../apps/cli/scripts/bundle-externals.ts"; +import { compileOptions } from "../../apps/cli/scripts/compile-options.ts"; const PORT = 4873; const REGISTRY = `http://localhost:${PORT}`; @@ -191,7 +192,19 @@ async function main() { const libc = libcForBunTarget(platform.bunTarget); console.log("[1/3] Compiling CLI binary..."); - await $`bun build ${entrypoint} --compile --target=${platform.bunTarget} --define=SUPABASE_LIBC=${JSON.stringify(libc)} --outfile=${bunBinary} ${oxfmtExternalArgs}`; + const buildResult = await Bun.build({ + entrypoints: [entrypoint], + compile: { target: platform.bunTarget, outfile: bunBinary }, + ...compileOptions, + external: [...OXFMT_OPTIONAL_PLUGIN_EXTERNALS], + define: { SUPABASE_LIBC: JSON.stringify(libc) }, + }); + for (const log of buildResult.logs) { + console.warn(log); + } + if (!buildResult.success) { + throw new Error("failed to compile the CLI binary"); + } { const goBinary = path.join(tmpPlatformBinDir, `supabase-go${platform.ext}`); From 3be8efc7c3c83ef9ba9702d9d4d2bd70c3dab341 Mon Sep 17 00:00:00 2001 From: avallete Date: Tue, 22 Sep 2026 16:25:29 +0200 Subject: [PATCH 04/20] perf(stack): clone snapshot directories with one cp on macOS A Postgres data directory is mostly small files, so one clone is cheaper than a clone per file. Unsupported entries still use the per-file copy. --- packages/stack/src/storage/DirectoryCopy.ts | 68 ++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/packages/stack/src/storage/DirectoryCopy.ts b/packages/stack/src/storage/DirectoryCopy.ts index c6f3746f57..222f04582d 100644 --- a/packages/stack/src/storage/DirectoryCopy.ts +++ b/packages/stack/src/storage/DirectoryCopy.ts @@ -1,8 +1,11 @@ // oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no lstat or clone-copy operation. +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no lstat or clone-copy operation. import { copyFile as nativeCopyFile, lstat } from "node:fs/promises"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no clone-copy flags. import { constants as fsConstants } from "node:fs"; -import { Effect, FileSystem, Path, Schema } from "effect"; +import { Effect, Exit, FileSystem, Path, Schema } from "effect"; export class DirectoryCopyError extends Schema.TaggedError()( "DirectoryCopyError", @@ -174,6 +177,69 @@ export const copyDirectory = Effect.fn("DirectoryCopy.copyDirectory")(function* onSuccess: () => Effect.fail(errorFor("validate", source, destination, "destination exists")), }), ); + // A PostgreSQL data directory is mostly small files. One clone is cheaper than a clone syscall per file. + if (process.platform === "darwin" && !(yield* findUnsupportedEntry(source, destination))) { + const cloned = yield* Effect.exit(cloneDirectoryWithCp(source, destination)); + if (Exit.isSuccess(cloned)) return; + yield* fs.remove(destination, { recursive: true, force: true }).pipe(Effect.ignore); + } yield* validateTree(source, destination, fs, path); return yield* copyTree(source, destination, fs, path); }); + +const execFileText = promisify(execFile); + +const runExec = ( + command: string, + args: ReadonlyArray, + source: string, + destination: string, + operation: string, +): Effect.Effect => + Effect.uninterruptible( + Effect.tryPromise({ + try: () => + execFileText(command, [...args], { encoding: "utf8" }).then((result) => + typeof result === "string" ? result : result.stdout, + ), + catch: (cause) => errorFor(operation, source, destination, cause), + }), + ); + +const findUnsupportedEntry = ( + source: string, + destination: string, +): Effect.Effect => + runExec( + "find", + [ + source, + "(", + "-type", + "l", + "-o", + "-type", + "p", + "-o", + "-type", + "s", + "-o", + "-type", + "b", + "-o", + "-type", + "c", + ")", + "-print", + "-quit", + ], + source, + destination, + "validate", + ).pipe(Effect.map((stdout) => stdout.trim().length > 0)); + +const cloneDirectoryWithCp = ( + source: string, + destination: string, +): Effect.Effect => + runExec("cp", ["-cRp", source, destination], source, destination, "clone").pipe(Effect.asVoid); From 5d46e26b10884e51c647ef69cb3fcfae2d4e3b8e Mon Sep 17 00:00:00 2001 From: avallete Date: Tue, 22 Sep 2026 18:56:43 +0200 Subject: [PATCH 05/20] perf(stack): copy snapshot directories with one host command Linux reflinks when the filesystem can, and Windows uses robocopy. A failed or interrupted host copy removes the partial tree first. --- packages/stack/src/services/Database.ts | 1 + .../DatabaseSnapshot.integration.test.ts | 11 +- .../stack/src/services/DatabaseSnapshot.ts | 3 + .../storage/DirectoryCopy.integration.test.ts | 88 ++++++- packages/stack/src/storage/DirectoryCopy.ts | 230 ++++++++++++++++-- 5 files changed, 306 insertions(+), 27 deletions(-) diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index da4ed5591a..130326ebb7 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -511,6 +511,7 @@ export const makeDatabase = ( Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path), Effect.provideService(Crypto.Crypto, crypto), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), Effect.mapError((cause) => errorFor("snapshot", cause)), ); diff --git a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts index 498aabc837..d03797aed1 100644 --- a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts +++ b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts @@ -1,6 +1,7 @@ import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Clock, Crypto, Effect, Exit, FileSystem, Option, Path, Schema, Scope } from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; import { makeDatabaseSnapshots } from "./DatabaseSnapshot.ts"; const version = "17.6.1.173"; @@ -56,7 +57,15 @@ const entryPath = (path: Path.Path, cache: string, name: string) => path.join(cache, "stack-database-snapshots", "entries", name); const live = ( - effect: Effect.Effect, + effect: Effect.Effect< + A, + E, + | ChildProcessSpawner.ChildProcessSpawner + | Crypto.Crypto + | FileSystem.FileSystem + | Path.Path + | Scope.Scope + >, ) => Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); describe("managed native database snapshots", () => { diff --git a/packages/stack/src/services/DatabaseSnapshot.ts b/packages/stack/src/services/DatabaseSnapshot.ts index 594ea74c8c..d4a951b8a2 100644 --- a/packages/stack/src/services/DatabaseSnapshot.ts +++ b/packages/stack/src/services/DatabaseSnapshot.ts @@ -13,6 +13,7 @@ import { Schedule, Schema, } from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; import { postgresVersion } from "../Artifacts.ts"; import { copyDirectory } from "../storage/DirectoryCopy.ts"; import type { DatabaseRuntime } from "./Database.ts"; @@ -67,6 +68,7 @@ export const makeDatabaseSnapshots = Effect.fn("DatabaseSnapshot.make")(function const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const version = postgresVersion(options.version); const root = path.join(options.cacheRoot, "stack-database-snapshots"); const entries = path.join(root, "entries"); @@ -149,6 +151,7 @@ export const makeDatabaseSnapshots = Effect.fn("DatabaseSnapshot.make")(function copyDirectory(source, destination).pipe( Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), Effect.mapError((cause) => errorFor("copy", cause)), ); diff --git a/packages/stack/src/storage/DirectoryCopy.integration.test.ts b/packages/stack/src/storage/DirectoryCopy.integration.test.ts index 1fdffc3178..64ed67766a 100644 --- a/packages/stack/src/storage/DirectoryCopy.integration.test.ts +++ b/packages/stack/src/storage/DirectoryCopy.integration.test.ts @@ -1,10 +1,18 @@ +import { chmodSync, mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; -import { Effect, Exit, FileSystem, Path, Scope } from "effect"; +import { Deferred, Effect, Exit, Fiber, FileSystem, Path, Scope, Sink, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { copyDirectory } from "./DirectoryCopy.ts"; -const run = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); +const run = ( + effect: Effect.Effect< + A, + E, + ChildProcessSpawner.ChildProcessSpawner | FileSystem.FileSystem | Path.Path | Scope.Scope + >, +) => Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); describe("copyDirectory", () => { it.live("copies nested files and preserves modes", () => @@ -78,6 +86,24 @@ describe("copyDirectory", () => { ), ); + it.live("rejects a file source and leaves the destination absent", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "directory-copy-file-" }); + const source = path.join(root, "source.txt"); + const destination = path.join(root, "destination"); + yield* fs.writeFileString(source, "file\n"); + + const result = yield* copyDirectory(source, destination).pipe(Effect.exit); + + expect(Exit.isFailure(result)).toBe(true); + expect(yield* fs.exists(destination)).toBe(false); + }), + ), + ); + it.live("rejects an existing destination", () => run( Effect.gen(function* () { @@ -121,4 +147,60 @@ describe("copyDirectory", () => { }), ), ); + + it.live("removes a partial copy when the host copy is interrupted", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "directory-copy-interrupt-" }); + const source = path.join(root, "source"); + const destination = path.join(root, "destination"); + const started = yield* Deferred.make(); + yield* fs.makeDirectory(source); + yield* fs.writeFileString(path.join(source, "file.txt"), "file\n"); + + const fiber = yield* copyDirectory(source, destination).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, hangingCopy(started)), + Effect.forkChild, + ); + yield* Deferred.await(started); + yield* Fiber.interrupt(fiber); + + expect(yield* fs.exists(destination)).toBe(false); + }), + ), + ); }); + +const processHandle = (exitCode: Effect.Effect) => + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + exitCode, + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + stdout: Stream.empty, + stderr: Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + +const hangingCopy = (started: Deferred.Deferred) => + ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command)) return Effect.die("unexpected piped command"); + if (command.command === "find") { + return Effect.succeed(processHandle(Effect.succeed(ChildProcessSpawner.ExitCode(0)))); + } + const destination = command.args.at(-1); + if (destination === undefined) return Effect.die("missing copy destination"); + return Effect.sync(() => { + const nested = join(destination, "nested"); + mkdirSync(nested, { recursive: true }); + writeFileSync(join(nested, "child.txt"), "partial\n"); + if (process.platform !== "win32") chmodSync(nested, 0o555); + return processHandle(Effect.never); + }).pipe(Effect.tap(() => Deferred.succeed(started, undefined))); + }); diff --git a/packages/stack/src/storage/DirectoryCopy.ts b/packages/stack/src/storage/DirectoryCopy.ts index 222f04582d..d6f7c898ac 100644 --- a/packages/stack/src/storage/DirectoryCopy.ts +++ b/packages/stack/src/storage/DirectoryCopy.ts @@ -1,11 +1,9 @@ // oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no lstat or clone-copy operation. -import { execFile } from "node:child_process"; -import { promisify } from "node:util"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no lstat or clone-copy operation. -import { copyFile as nativeCopyFile, lstat } from "node:fs/promises"; +import { copyFile as nativeCopyFile, lstat, readdir } from "node:fs/promises"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no clone-copy flags. import { constants as fsConstants } from "node:fs"; -import { Effect, Exit, FileSystem, Path, Schema } from "effect"; +import { Cause, Effect, FileSystem, Option, Path, Schema, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; export class DirectoryCopyError extends Schema.TaggedError()( "DirectoryCopyError", @@ -177,39 +175,97 @@ export const copyDirectory = Effect.fn("DirectoryCopy.copyDirectory")(function* onSuccess: () => Effect.fail(errorFor("validate", source, destination, "destination exists")), }), ); - // A PostgreSQL data directory is mostly small files. One clone is cheaper than a clone syscall per file. - if (process.platform === "darwin" && !(yield* findUnsupportedEntry(source, destination))) { - const cloned = yield* Effect.exit(cloneDirectoryWithCp(source, destination)); - if (Exit.isSuccess(cloned)) return; - yield* fs.remove(destination, { recursive: true, force: true }).pipe(Effect.ignore); + // cp and robocopy keep or follow links. Only a plain directory tree can use them. + const copies = hostCopies(source, destination); + if (copies.length > 0 && (yield* directoryTreeIsCopyable(source, destination, path))) { + for (const copy of copies) { + const copied = yield* runExec( + copy.command, + copy.args, + source, + destination, + "copy", + copy.acceptStatus, + ).pipe( + Effect.asVoid, + // matchCauseEffect does not observe an external interrupt. + Effect.onInterrupt(() => removePartial(destination, fs, path)), + Effect.matchCauseEffect({ + onSuccess: () => Effect.succeed("copied" as const), + onFailure: (cause) => + removePartial(destination, fs, path).pipe( + Effect.flatMap(() => { + if (Cause.hasInterrupts(cause)) return Effect.failCause(cause); + const outcome = isUsageFailure(cause) ? "usage" : "failed"; + return Effect.succeed(outcome); + }), + ), + }), + ); + if (copied === "copied") return; + if (copied === "failed") break; + } } yield* validateTree(source, destination, fs, path); return yield* copyTree(source, destination, fs, path); }); -const execFileText = promisify(execFile); - const runExec = ( command: string, args: ReadonlyArray, source: string, destination: string, operation: string, -): Effect.Effect => - Effect.uninterruptible( - Effect.tryPromise({ - try: () => - execFileText(command, [...args], { encoding: "utf8" }).then((result) => - typeof result === "string" ? result : result.stdout, - ), - catch: (cause) => errorFor(operation, source, destination, cause), + acceptStatus: (status: number) => boolean = () => false, +): Effect.Effect => + Effect.scoped( + Effect.gen(function* () { + const child = yield* ChildProcess.make(command, [...args], { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }).pipe(Effect.mapError((cause) => errorFor(operation, source, destination, cause))); + const [text, stderr, exitCode] = yield* Effect.all( + [ + child.stdout.pipe(Stream.decodeText, Stream.mkString), + child.stderr.pipe(Stream.decodeText, Stream.mkString), + child.exitCode, + ], + { concurrency: 3 }, + ).pipe(Effect.mapError((cause) => errorFor(operation, source, destination, cause))); + const status = Number(exitCode); + if (status !== 0 && !acceptStatus(status)) { + return yield* errorFor(operation, source, destination, new Error(stderr)); + } + return text; }), ); +const directoryTreeIsCopyable = ( + source: string, + destination: string, + path: Path.Path, +): Effect.Effect => + Effect.gen(function* () { + const stats = yield* inspect(source, destination); + if (!stats.isDirectory() || stats.isSymbolicLink()) return false; + return yield* ( + process.platform === "win32" + ? scanUnsupported(source, destination, path) + : findUnsupportedEntry(source, destination) + ).pipe( + Effect.matchCauseEffect({ + onSuccess: (found) => Effect.succeed(!found), + onFailure: (cause) => + Cause.hasInterrupts(cause) ? Effect.interrupt : Effect.succeed(false), + }), + ); + }); + const findUnsupportedEntry = ( source: string, destination: string, -): Effect.Effect => +): Effect.Effect => runExec( "find", [ @@ -238,8 +294,136 @@ const findUnsupportedEntry = ( "validate", ).pipe(Effect.map((stdout) => stdout.trim().length > 0)); -const cloneDirectoryWithCp = ( +// Windows find is a content search, so the unsupported-entry scan stays in process. +const scanUnsupported = ( source: string, destination: string, + path: Path.Path, +): Effect.Effect => + Effect.gen(function* () { + const pending = [source]; + let current = pending.pop(); + while (current !== undefined) { + const directory = current; + const entries = yield* Effect.tryPromise({ + try: () => readdir(directory, { withFileTypes: true }), + catch: (cause) => errorFor("validate", source, destination, cause), + }); + for (const entry of entries) { + if (entry.isSymbolicLink() || (!entry.isDirectory() && !entry.isFile())) return true; + if (entry.isDirectory()) pending.push(path.join(directory, entry.name)); + } + current = pending.pop(); + } + return false; + }); + +interface HostCopy { + readonly command: string; + readonly args: ReadonlyArray; + readonly acceptStatus: (status: number) => boolean; +} + +const rejectStatus = (_status: number) => false; + +// A preserved directory mode can omit user write, and unlink then fails with EACCES. +const grantDirectoryWrite = ( + directory: string, + fs: FileSystem.FileSystem, + path: Path.Path, +): Effect.Effect => + Effect.gen(function* () { + const info = yield* fs + .stat(directory) + .pipe(Effect.mapError((cause) => errorFor("chmod", directory, directory, cause))); + yield* fs + .chmod(directory, (Number(info.mode) | 0o700) & 0o7777) + .pipe(Effect.mapError((cause) => errorFor("chmod", directory, directory, cause))); + if (info.type !== "Directory") return; + const names = yield* fs + .readDirectory(directory) + .pipe(Effect.mapError((cause) => errorFor("readDirectory", directory, directory, cause))); + for (const name of names) { + const child = path.join(directory, name); + const childInfo = yield* fs + .stat(child) + .pipe(Effect.mapError((cause) => errorFor("chmod", directory, child, cause))); + if (childInfo.type === "Directory") yield* grantDirectoryWrite(child, fs, path); + } + }); + +const removePartial = ( + destination: string, + fs: FileSystem.FileSystem, + path: Path.Path, ): Effect.Effect => - runExec("cp", ["-cRp", source, destination], source, destination, "clone").pipe(Effect.asVoid); + Effect.gen(function* () { + if ( + !(yield* fs + .exists(destination) + .pipe(Effect.mapError((cause) => errorFor("remove", destination, destination, cause)))) + ) + return; + yield* grantDirectoryWrite(destination, fs, path); + yield* fs + .remove(destination, { recursive: true, force: true }) + .pipe(Effect.mapError((cause) => errorFor("remove", destination, destination, cause))); + }); + +const isUsageError = (cause: unknown): boolean => { + if (typeof cause !== "object" || cause === null || !("message" in cause)) return false; + const message = cause.message; + return ( + typeof message === "string" && + /unrecognized option|invalid option|unknown option|illegal option/iu.test(message) + ); +}; + +const isUsageFailure = (cause: Cause.Cause): boolean => + Option.match(Cause.findErrorOption(cause), { + onNone: () => false, + onSome: (error) => isUsageError(error.cause), + }); + +// One process copies the tree. macOS clones, and Linux reflinks when the filesystem can. +const hostCopies = (source: string, destination: string): ReadonlyArray => { + switch (process.platform) { + case "darwin": + return [{ command: "cp", args: ["-cRp", source, destination], acceptStatus: rejectStatus }]; + case "linux": + return [ + { + command: "cp", + args: ["-R", "--preserve=mode", "--reflink=auto", "--", source, destination], + acceptStatus: rejectStatus, + }, + // BusyBox cp has no reflink flag. -Rp still copies the tree in one process. + { command: "cp", args: ["-Rp", source, destination], acceptStatus: rejectStatus }, + ]; + case "win32": + // Robocopy's exit status is a bit field. Values below 8 mean the copy succeeded. + return [ + { + command: "robocopy", + args: [ + source, + destination, + "/E", + "/COPY:DAT", + "/DCOPY:DAT", + "/MT:8", + "/R:0", + "/W:0", + "/NFL", + "/NDL", + "/NJH", + "/NJS", + "/NP", + ], + acceptStatus: (status) => status < 8, + }, + ]; + default: + return []; + } +}; From b4c9db8bd312ad1b7670ec7322e253cc73d719e7 Mon Sep 17 00:00:00 2001 From: avallete Date: Tue, 22 Sep 2026 16:25:35 +0200 Subject: [PATCH 06/20] perf(stack): stop shadow databases without waiting Shadow destroy issues docker stop and rm in the background. An explicit stop, including the one before a snapshot, still waits for a checkpoint. --- apps/cli/src/command-internal/shadow-phase.ts | 13 ++++ apps/cli/src/command-internal/stack-shadow.ts | 71 +++++++++++++------ .../db/shared/pgdelta-engine.next.layer.ts | 10 ++- packages/stack/src/Orchestrator.ts | 3 +- packages/stack/src/Service.ts | 15 +++- packages/stack/src/runtime/Container.ts | 53 +++++++++++++- packages/stack/src/services/Database.ts | 38 +++++++--- packages/stack/src/services/shadowPhase.ts | 16 +++++ .../src/storage/DockerDatabaseStorage.ts | 7 ++ 9 files changed, 190 insertions(+), 36 deletions(-) create mode 100644 apps/cli/src/command-internal/shadow-phase.ts create mode 100644 packages/stack/src/services/shadowPhase.ts diff --git a/apps/cli/src/command-internal/shadow-phase.ts b/apps/cli/src/command-internal/shadow-phase.ts new file mode 100644 index 0000000000..8a658c934d --- /dev/null +++ b/apps/cli/src/command-internal/shadow-phase.ts @@ -0,0 +1,13 @@ +// oxlint-disable-next-line effecttsgo/node-builtin-import -- phase file is a CLI-side probe; FileSystem has no append. +import { appendFileSync } from "node:fs"; +import { Effect } from "effect"; + +/** Appends a timestamp when `SUPABASE_SHADOW_PHASE_FILE` is set. */ +export const shadowPhase = (label: string) => + Effect.suspend(() => { + const file = process.env["SUPABASE_SHADOW_PHASE_FILE"]; + if (file === undefined || file.length === 0) return Effect.void; + return Effect.try(() => { + appendFileSync(file, `${Date.now()} ${process.pid} ${label}\n`); + }).pipe(Effect.ignore); + }); diff --git a/apps/cli/src/command-internal/stack-shadow.ts b/apps/cli/src/command-internal/stack-shadow.ts index 52e6dd6eff..35285c6ba3 100644 --- a/apps/cli/src/command-internal/stack-shadow.ts +++ b/apps/cli/src/command-internal/stack-shadow.ts @@ -17,6 +17,7 @@ import { import type { SetupDatabaseOptions } from "./db-bootstrap/db-setup.ts"; import { listLocalMigrationPaths } from "./migration-history.ts"; import { applyMigrations } from "./migration-apply.ts"; +import { shadowPhase } from "./shadow-phase.ts"; import { stackShadowCacheEntry, stackShadowCacheRoles } from "./stack-shadow-cache.ts"; type Runtime = "native" | "docker" | "podman"; @@ -57,12 +58,14 @@ const acquireNamespace = Effect.fn("StackShadow.acquireNamespace")(function* (op const runtimeInfo = yield* RuntimeInfo; const runtime = opts.runtime ?? (yield* stackProjectRuntime) ?? defaultStackRuntime(runtimeInfo); const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-shadow-" }); + yield* shadowPhase("namespace-begin"); const stack = yield* api.create({ projectRoot: root, stateRoot: path.join(settings.supabaseHome, "stacks"), cacheRoot: path.join(settings.supabaseHome, "cache", "stack"), runtime, }); + yield* shadowPhase("namespace-end"); return { stack, runtime }; }); @@ -89,12 +92,17 @@ const initialize = Effect.fn("StackShadow.initialize")(function* ( jwtSecret: Redacted.make(input.jwtSecret), jwtExpiry: input.jwtExpiry, healthTimeoutMs: input.healthTimeoutSeconds * 1_000, + // Destroy issues stop without waiting. The stop before a snapshot still checkpoints. + stopGraceSeconds: 0, ...(input.rootKey === undefined ? {} : { rootKey: Redacted.make(input.rootKey) }), settings: dbSettings, }, endpoints: { sql: { port: opts.port ?? "auto" } }, }); + yield* shadowPhase("create-db-begin"); let database = yield* createDatabase(); + yield* shadowPhase("create-db-end"); + yield* shadowPhase("cache-begin"); const cacheResolution = yield* Effect.result( stackShadowCacheEntry( input, @@ -105,8 +113,18 @@ const initialize = Effect.fn("StackShadow.initialize")(function* ( opts.bypassCache, ), ); + yield* shadowPhase("cache-end"); const output = yield* Output; const cache = Result.isSuccess(cacheResolution) ? cacheResolution.success : undefined; + const startReady = (db: DatabaseInstance) => + Effect.gen(function* () { + yield* shadowPhase("start-begin"); + yield* db.start; + yield* shadowPhase("start-end"); + yield* shadowPhase("ready-begin"); + yield* db.ready; + yield* shadowPhase("ready-end"); + }); if (Result.isFailure(cacheResolution)) yield* output.raw( `Warning: stack shadow cache unavailable: ${causeMessage(cacheResolution.failure)}; continuing uncached.\n`, @@ -115,32 +133,26 @@ const initialize = Effect.fn("StackShadow.initialize")(function* ( let restored = false; if (cache !== undefined) { const warmAttempt = yield* Effect.result( - database - .restoreSnapshot(cache.key) - .pipe( - Effect.flatMap((restoredSnapshot) => - restoredSnapshot - ? database.start.pipe(Effect.andThen(database.ready), Effect.as(true)) - : Effect.succeed(false), - ), - ), + Effect.gen(function* () { + yield* shadowPhase("restore-begin"); + const restoredSnapshot = yield* database.restoreSnapshot(cache.key); + yield* shadowPhase("restore-end"); + if (!restoredSnapshot) return false; + yield* startReady(database); + return true; + }), ); if (Result.isSuccess(warmAttempt)) { restored = warmAttempt.success; - if (!restored) { - yield* database.start; - yield* database.ready; - } + if (!restored) yield* startReady(database); } else { yield* output.raw("Warning: cached stack shadow baseline unusable; recreating.\n", "stderr"); yield* database.destroy; database = yield* createDatabase(); - yield* database.start; - yield* database.ready; + yield* startReady(database); } } else { - yield* database.start; - yield* database.ready; + yield* startReady(database); } const catalog = yield* StackCatalogSetup; if (!restored) @@ -176,7 +188,7 @@ const initialize = Effect.fn("StackShadow.initialize")(function* ( const published = yield* Effect.result( database.stop.pipe(Effect.flatMap(() => database.saveSnapshot(cache.key))), ); - yield* database.start.pipe(Effect.andThen(database.ready)); + yield* startReady(database); if (Result.isSuccess(published)) { snapshotKey = cache.key; } else { @@ -187,7 +199,9 @@ const initialize = Effect.fn("StackShadow.initialize")(function* ( } } } + yield* shadowPhase("credentials-begin"); const credentials = yield* database.credentials({ from: "host" }); + yield* shadowPhase("credentials-end"); const credentialUrl = credentials.databaseUrl; const credentialsConn = credentialUrl === undefined ? undefined : parseConnectionString(credentialUrl); @@ -218,13 +232,18 @@ export const stackAcquireShadowDatabase = Effect.fn("StackShadow.acquire")(funct ) { const output = yield* Output; const namespace = yield* Effect.acquireRelease(acquireNamespace(opts), ({ stack }) => - stack.destroy.pipe( - Effect.catch((cause) => - output.raw( - `Failed to destroy shadow stack ${stack.id}: ${cause.message}. Run supabase stack destroy --stack-id ${stack.id} to remove it.\n`, - "stderr", + shadowPhase("destroy-begin").pipe( + Effect.andThen( + stack.destroy.pipe( + Effect.catch((cause) => + output.raw( + `Failed to destroy shadow stack ${stack.id}: ${cause.message}. Run supabase stack destroy --stack-id ${stack.id} to remove it.\n`, + "stderr", + ), + ), ), ), + Effect.ensuring(shadowPhase("destroy-end")), ), ); return yield* initialize(namespace.stack, namespace.runtime, input, opts); @@ -243,6 +262,7 @@ export const stackMigrateShadow = Effect.fn("StackShadow.migrate")(function* ( handle: StackShadowAcquiredHandle, input: ShadowSetupInput, ) { + yield* shadowPhase("migrations-list-begin"); const migrationsDir = input.path.join(input.workdir, "supabase", "migrations"); const pending = yield* listLocalMigrationPaths(input.fs, input.path, migrationsDir).pipe( Effect.mapError((cause) => new ShadowDbError({ message: cause.message, reason: "filesystem" })), @@ -250,7 +270,11 @@ export const stackMigrateShadow = Effect.fn("StackShadow.migrate")(function* ( const conn = parseConnectionString(handle.url); if (conn === undefined) return yield* new ShadowDbError({ message: "Invalid shadow database URL", reason: "connect" }); + yield* shadowPhase("migrations-list-end"); + yield* shadowPhase("connect-begin"); const session = yield* connectShadowDatabase(conn); + yield* shadowPhase("connect-end"); + yield* shadowPhase("apply-begin"); yield* applyMigrations( session, input.fs, @@ -263,6 +287,7 @@ export const stackMigrateShadow = Effect.fn("StackShadow.migrate")(function* ( ), Effect.mapError(shadowError), ); + yield* shadowPhase("apply-end"); }); export const stackPrepareShadowSource = Effect.fn("StackShadow.prepareSource")( diff --git a/apps/cli/src/commands/db/shared/pgdelta-engine.next.layer.ts b/apps/cli/src/commands/db/shared/pgdelta-engine.next.layer.ts index 58bcc00c36..b019a6dc57 100644 --- a/apps/cli/src/commands/db/shared/pgdelta-engine.next.layer.ts +++ b/apps/cli/src/commands/db/shared/pgdelta-engine.next.layer.ts @@ -36,6 +36,7 @@ import { reportPgDeltaNextDiagnostics, } from "./pgdelta-next-diagnostics.ts"; import { PgDeltaNextShadow } from "./pgdelta-next-shadow.service.ts"; +import { shadowPhase } from "../../../command-internal/shadow-phase.ts"; function pgDeltaNextConnectSuggestion(cause: unknown): string | undefined { if (cause instanceof DbConnectError) return cause.suggestion; @@ -291,9 +292,16 @@ export const pgDeltaNextEngineLayer = Layer.effect( diffDatabase: (input) => Effect.scoped( Effect.gen(function* () { + yield* shadowPhase("diff-source-begin"); const migrationsPool = yield* acquireDatabase(input.source, input.context.projectEnv); + yield* shadowPhase("diff-source-end"); + yield* shadowPhase("diff-target-begin"); const desiredPool = yield* acquireDatabase(input.target, input.context.projectEnv); - return yield* diffPools(input, migrationsPool, desiredPool); + yield* shadowPhase("diff-target-end"); + yield* shadowPhase("diff-adapter-begin"); + const result = yield* diffPools(input, migrationsPool, desiredPool); + yield* shadowPhase("diff-adapter-end"); + return result; }), ).pipe(Effect.mapError(pgDeltaNextEngineError)), exportDeclarativeSchema: (input) => diff --git a/packages/stack/src/Orchestrator.ts b/packages/stack/src/Orchestrator.ts index 3768183a61..00c51bb87b 100644 --- a/packages/stack/src/Orchestrator.ts +++ b/packages/stack/src/Orchestrator.ts @@ -700,7 +700,8 @@ const makeOrchestrator = Effect.gen(function* () { }); const destroyNamespace = Effect.fn("Orchestrator.destroyNamespace")(function* () { const values = yield* Ref.get(registry); - yield* stopIds([...values.keys()]); + // Each destroy stops that service. A prior waited stop would block ephemeral + // databases that only need the stop command to be issued. const configured = yield* configurationGraph(yield* Ref.get(composition)); for (const id of topo([...values.keys()], configured.prerequisites).toReversed()) yield* destroyNode(id); diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index 5ee3f910e4..87c56aafdd 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -12,6 +12,7 @@ import { SubscriptionRef, } from "effect"; import type { Stream } from "effect"; +import { shadowPhase } from "./services/shadowPhase.ts"; type ServiceLifecycle = "stopped" | "starting" | "running" | "stopping"; type ServiceHealth = "starting" | "healthy" | "unhealthy"; @@ -65,6 +66,8 @@ export interface RuntimeSession { /** Resolves with the runtime's terminal result and remains attached to this session. */ readonly exit: Effect.Effect>; readonly stop: Effect.Effect; + /** Skips a clean checkpoint. Used when the data directory is about to be deleted. */ + readonly discard?: Effect.Effect; readonly remove: Effect.Effect; } @@ -227,6 +230,7 @@ export const makeService = ( const stopNow = Effect.fn("Service.stopNow")(function* ( expected?: SessionRecord, retainWake = false, + discard = false, ) { yield* Effect.gen(function* () { const record = yield* Ref.get(current); @@ -243,7 +247,14 @@ export const makeService = ( ); yield* Scope.close(record.healthScope, Exit.void); if (!(yield* Ref.get(record.stopped))) { - yield* record.runtime.stop.pipe( + yield* shadowPhase( + `stopNow discard=${String(discard)} has=${String(record.runtime.discard !== undefined)}`, + ); + const halt = + discard && record.runtime.discard !== undefined + ? record.runtime.discard + : record.runtime.stop; + yield* halt.pipe( Effect.tapError((error) => SubscriptionRef.update(observations, (value) => ({ ...value, @@ -601,7 +612,7 @@ export const makeService = ( yield* coordinate("destroy", invalidate()); yield* setOperation("destroy"); yield* Effect.gen(function* () { - yield* stopNow(); + yield* stopNow(undefined, false, true); { const dataScope = yield* Scope.fork(owner, "parallel"); yield* definition diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index 82c62475b0..3875071279 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -16,6 +16,7 @@ import { Stream, } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { shadowPhase } from "../services/shadowPhase.ts"; export class ContainerError extends Data.TaggedError("ContainerError")<{ readonly operation: string; @@ -39,6 +40,8 @@ interface ContainerSpec { }>; readonly workingDir?: string; readonly ports?: ReadonlyArray; + /** Seconds `docker stop` waits before SIGKILL. Omitted means 10. */ + readonly stopGraceSeconds?: number; } export interface ContainerProcess { @@ -51,6 +54,9 @@ export interface ContainerProcess { readonly exitCode: Effect.Effect; readonly stdin: Sink.Sink; readonly stop: Effect.Effect; + /** Starts `stop` and a later `rm` without waiting for either to finish. */ + readonly beginStop: Effect.Effect; + readonly kill: Effect.Effect; readonly remove: Effect.Effect; } @@ -251,7 +257,50 @@ export const makeContainerRuntime = (options: { const removed = yield* Ref.make(false); const stop = Effect.gen(function* () { if ((yield* Ref.get(removed)) || (yield* Ref.get(stopped))) return; - yield* run(["stop", "--time", "10", id]); + yield* shadowPhase("container-stop"); + const grace = + spec.stopGraceSeconds !== undefined && + Number.isInteger(spec.stopGraceSeconds) && + spec.stopGraceSeconds > 0 && + spec.stopGraceSeconds <= 60 + ? String(spec.stopGraceSeconds) + : "10"; + yield* run(["stop", "--time", grace, id]); + yield* Ref.set(stopped, true); + }); + // Issues stop, then removes the container, without waiting. The caller continues. + const beginStop = Effect.scoped( + Effect.gen(function* () { + if ((yield* Ref.get(removed)) || (yield* Ref.get(stopped))) return; + yield* shadowPhase("container-stop-background"); + const script = `${options.engine} stop --time 10 ${id} && ${options.engine} rm ${id}`; + const child = yield* Effect.mapError( + spawner.spawn( + ChildProcess.make( + process.platform === "win32" ? "cmd" : "sh", + process.platform === "win32" ? ["/d", "/c", script] : ["-c", script], + { + detached: true, + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + }, + ), + ), + (cause) => errorFor("stop", cause), + ); + // `unref` succeeds with a reref effect. Drop it so the stop outlives this scope. + yield* Effect.asVoid( + Effect.mapError(child.unref, (cause) => errorFor("stop", cause)), + ); + yield* Ref.set(stopped, true); + yield* Ref.set(removed, true); + }), + ); + const kill = Effect.gen(function* () { + if ((yield* Ref.get(removed)) || (yield* Ref.get(stopped))) return; + yield* shadowPhase("container-kill"); + yield* run(["kill", id]); yield* Ref.set(stopped, true); }); const remove = Effect.gen(function* () { @@ -274,6 +323,8 @@ export const makeContainerRuntime = (options: { exitCode: Effect.fail(errorFor("wait", "Container did not start")), stdin: Sink.fail(errorFor("stdin", "Container did not start")), stop, + beginStop, + kill, remove, }; let owned = partial; diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index 130326ebb7..6c882e6288 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -25,6 +25,7 @@ import { resolveArtifact, type PreparedNativeArtifact, } from "../Artifacts.ts"; +import { shadowPhase } from "./shadowPhase.ts"; import { makeContainerRuntime, type ContainerProcess, @@ -61,6 +62,8 @@ export const DatabaseConfig = Schema.Struct({ jwtSecret: Schema.Redacted(Schema.String), jwtExpiry: Schema.Finite, healthTimeoutMs: Schema.optionalKey(Schema.Finite), + /** When 0, destroy issues stop in the background. An explicit stop still waits. */ + stopGraceSeconds: Schema.optionalKey(Schema.Finite), rootKey: Schema.optionalKey(Schema.Redacted(Schema.String)), settings: Schema.optionalKey( Schema.Record(Schema.String, Schema.Union([Schema.String, Schema.Finite, Schema.Boolean])), @@ -237,8 +240,10 @@ const health = Effect.fn("Database.health")(( }), ); const retryProbe = probe.pipe(Effect.retry(Schedule.spaced("250 millis"))); - return reconcile.pipe( + return shadowPhase("health-probe-begin").pipe( + Effect.andThen(reconcile), Effect.andThen(retryProbe), + Effect.tap(() => shadowPhase("health-probe-ok")), Effect.andThen( Effect.scoped( Effect.gen(function* () { @@ -275,6 +280,7 @@ const health = Effect.fn("Database.health")(( }), ), ); + yield* shadowPhase("health-bootstrap-begin"); yield* ensureInternalDatabase(session, openInternal).pipe( Effect.mapError((cause) => errorFor("bootstrap", cause)), ); @@ -283,6 +289,7 @@ const health = Effect.fn("Database.health")(( jwtSecret: config.jwtSecret, jwtExpiry: config.jwtExpiry, }).pipe(Effect.mapError((cause) => errorFor("bootstrap", cause))); + yield* shadowPhase("health-bootstrap-end"); const readyMarker = yield* Schema.encodeEffect( Schema.fromJsonString(DatabaseReadyMarker), )({ @@ -333,10 +340,15 @@ const publishLogs = Effect.fn("Database.publishLogs")(( ); }); -const runtimeFromContainer = (process: ContainerProcess): RuntimeSession => ({ +const runtimeFromContainer = (process: ContainerProcess, discard: boolean): RuntimeSession => ({ health: Effect.void, exit: processExit(process.exitCode), stop: process.stop.pipe(Effect.mapError((cause) => errorFor("stop", cause))), + ...(discard + ? { + discard: process.beginStop.pipe(Effect.mapError((cause) => errorFor("stop", cause))), + } + : {}), remove: process.remove.pipe(Effect.mapError((cause) => errorFor("remove", cause))), }); @@ -750,10 +762,12 @@ export const makeDatabase = ( ), } satisfies RuntimeSession; } + yield* shadowPhase("launch-image-begin"); const image = yield* resolveArtifact({ service: "database", version: config.version, }).pipe(Effect.mapError((cause) => errorFor("launch", cause))); + yield* shadowPhase("launch-image-end"); const selectedContainer = container; if (selectedContainer === undefined) return yield* errorFor("launch", "Container runtime is unavailable"); @@ -762,8 +776,12 @@ export const makeDatabase = ( (yield* storage.needsDataChown.pipe( Effect.mapError((cause) => errorFor("launch", cause)), )) - ) + ) { + yield* shadowPhase("launch-chown-begin"); yield* dataCommand(config.version, ["chown", "100:101", "/var/lib/postgresql/data"]); + yield* shadowPhase("launch-chown-end"); + } + yield* shadowPhase("launch-container-begin"); const launched = yield* selectedContainer .launch({ image: image.image, @@ -793,13 +811,16 @@ export const makeDatabase = ( ]), ], ports: [5432], + ...(config.stopGraceSeconds === undefined + ? {} + : { stopGraceSeconds: config.stopGraceSeconds }), }) .pipe( Effect.catchTag("ContainerLaunchError", ({ failure, process }) => Effect.fail( new ServiceLaunchError({ failure: errorFor("launch", failure), - runtime: runtimeFromContainer(process), + runtime: runtimeFromContainer(process, false), }), ), ), @@ -808,14 +829,17 @@ export const makeDatabase = ( ), Scope.provide(context.scope), ); + yield* shadowPhase("launch-container-end"); const port = launched.ports[5432]; if (port === undefined) return yield* errorFor("launch", "Container did not publish PostgreSQL"); const selectedEndpoint: BackendEndpoint = { kind: "tcp", host: "127.0.0.1", port }; yield* Ref.set(endpoint, selectedEndpoint); yield* publishLogs(launched, logs, context.scope); + yield* shadowPhase(`stop-grace-${String(config.stopGraceSeconds)}`); + const session = runtimeFromContainer(launched, config.stopGraceSeconds === 0); return { - ...runtimeFromContainer(launched), + ...session, health: health( selectedEndpoint, config, @@ -838,9 +862,7 @@ export const makeDatabase = ( .pipe(Effect.mapError((cause) => errorFor("health", cause))), }, ), - remove: runtimeFromContainer(launched).remove.pipe( - Effect.tap(() => Ref.set(endpoint, undefined)), - ), + remove: session.remove.pipe(Effect.tap(() => Ref.set(endpoint, undefined))), } satisfies RuntimeSession; }), ); diff --git a/packages/stack/src/services/shadowPhase.ts b/packages/stack/src/services/shadowPhase.ts new file mode 100644 index 0000000000..ea8b856980 --- /dev/null +++ b/packages/stack/src/services/shadowPhase.ts @@ -0,0 +1,16 @@ +// oxlint-disable-next-line effecttsgo/node-builtin-import -- phase file is a host-side probe; FileSystem has no append. +import { appendFileSync } from "node:fs"; +import { Effect } from "effect"; + +/** Appends a timestamp when `SUPABASE_SHADOW_PHASE_FILE` is set. */ +export const shadowPhase = (label: string) => + Effect.suspend(() => { + // Config and Clock would add ConfigError to service methods that call this probe. + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- service-free probe, see above. + const file = process.env["SUPABASE_SHADOW_PHASE_FILE"]; + if (file === undefined || file.length === 0) return Effect.void; + return Effect.try(() => { + // oxlint-disable-next-line effecttsgo/global-date-in-effect -- service-free probe, see above. + appendFileSync(file, `${Date.now()} ${process.pid} ${label}\n`); + }).pipe(Effect.ignore); + }); diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index 9c99261c92..c426aae589 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -15,6 +15,7 @@ import { ChildProcess } from "effect/unstable/process"; import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; import type { ContainerRuntime } from "../runtime/Container.ts"; import type { DatabaseRuntime } from "../services/Database.ts"; +import { shadowPhase } from "../services/shadowPhase.ts"; const HELPER_IMAGE = "public.ecr.aws/docker/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; @@ -449,7 +450,9 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* options.fs .makeDirectory(options.cacheRoot, { recursive: true }) .pipe(Effect.mapError((cause) => errorFor("helper", cause))); + yield* shadowPhase("helper-prepare-begin"); yield* options.container.prepare(HELPER_IMAGE); + yield* shadowPhase("helper-prepare-end"); const token = yield* options.crypto.randomUUIDv4.pipe( Effect.mapError((cause) => errorFor("helper", cause)), ); @@ -457,6 +460,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") // Register the deterministic owned name before the remote create starts so an // interrupted docker run can still be removed by the same scope. yield* Ref.set(helperId, name); + yield* shadowPhase("helper-run-begin"); const created = yield* engineCommand([ "run", "-d", @@ -476,6 +480,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ]); if (!/^[a-f0-9]{12,64}$/u.test(created)) return yield* errorFor("helper", "Docker returned an invalid helper identity"); + yield* shadowPhase("helper-run-end"); return name; }), ); @@ -830,10 +835,12 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") store.backend === "host" ? `; owner=$(stat -c "%u:%g" /cache); mkdir -p /cache/stack-database-snapshots-helper; chown "$owner" /cache/stack-database-snapshots-helper; chown -R "$owner" ${root}` : ""; + yield* shadowPhase("restore-helper-begin"); const result = yield* runHelper( `set -eu; mkdir -p ${root}; flock -x -w 120 ${shellQuote(`${root}/.lock`)} sh -eu -c ${shellQuote(`set -eu; mkdir -p ${root}/entries ${root}/stages; find ${root}/stages -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +; find ${root}/entries -mindepth 1 -maxdepth 1 -name '*.retired' -exec rm -rf -- {} +; trap 'rm -rf ${stage}${cacheOwnership}' EXIT; if ! ${targetSetup}; then echo 'Initialized restore target data directory is missing' >&2; exit 1; fi; bad=$(find ${data} -mindepth 1 -print -quit); if [ -n "$bad" ]; then echo NONEMPTY; exit 0; fi; if [ ! -d ${source} ]; then echo MISS; exit 0; fi; if [ ! -f ${source}/descriptor.json ]; then echo 'Snapshot descriptor is missing' >&2; exit 1; fi; actual=$(cat ${source}/descriptor.json); expected=${shellQuote(descriptor)}; if [ "$actual" != "$expected" ]; then echo 'Snapshot descriptor does not match requested identity' >&2; exit 1; fi; bad=$(find ${source}/data \\( ! -type f ! -type d \\) -print -quit); if [ -n "$bad" ]; then echo "Snapshot contains unsupported filesystem entry $bad" >&2; exit 1; fi; if [ -e ${source}/data/postmaster.pid ]; then echo 'Snapshot contains postmaster.pid' >&2; exit 1; fi; rm -rf ${stage}; cp -a --reflink=auto ${source}/data ${stage}; actual=$(cat ${stage}/PG_VERSION); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Snapshot PostgreSQL major does not match requested version' >&2; exit 1; fi; ${store.backend === "host" ? `chown -R 100:101 ${stage};` : ""} rmdir ${data}; mv ${stage} ${data}; touch ${source}; echo HIT`)}; rm -rf ${shellQuote(stage)}`, paths.mounts, ); + yield* shadowPhase("restore-helper-end"); if (result === "MISS") return false; if (result === "NONEMPTY") return yield* errorFor("restore", "Restore target data directory must be empty"); From e23634953d3d16eb5f8c67e6fa11b7528e97b5a4 Mon Sep 17 00:00:00 2001 From: avallete Date: Tue, 22 Sep 2026 20:40:02 +0200 Subject: [PATCH 07/20] perf(stack): shorten warm shadow database diffs Reuse one helper container per volume, remember the daemon identity, and turn crash safety off for a shadow that is deleted after the diff. --- apps/cli/src/command-internal/stack-shadow.ts | 5 +- packages/stack/src/Owner.ts | 3 + packages/stack/src/services/Catalog.ts | 1 + packages/stack/src/services/Database.ts | 206 +++++---- packages/stack/src/services/Recipe.ts | 3 + .../src/storage/DockerDatabaseStorage.ts | 418 +++++++++++++----- .../stack/src/storage/DockerHelperRegistry.ts | 82 ++++ .../storage/DockerHelperRegistry.unit.test.ts | 44 ++ 8 files changed, 572 insertions(+), 190 deletions(-) create mode 100644 packages/stack/src/storage/DockerHelperRegistry.ts create mode 100644 packages/stack/src/storage/DockerHelperRegistry.unit.test.ts diff --git a/apps/cli/src/command-internal/stack-shadow.ts b/apps/cli/src/command-internal/stack-shadow.ts index 35285c6ba3..a5b5a35b4e 100644 --- a/apps/cli/src/command-internal/stack-shadow.ts +++ b/apps/cli/src/command-internal/stack-shadow.ts @@ -146,7 +146,10 @@ const initialize = Effect.fn("StackShadow.initialize")(function* ( restored = warmAttempt.success; if (!restored) yield* startReady(database); } else { - yield* output.raw("Warning: cached stack shadow baseline unusable; recreating.\n", "stderr"); + yield* output.raw( + `Warning: cached stack shadow baseline unusable (${causeMessage(warmAttempt.failure)}); recreating.\n`, + "stderr", + ); yield* database.destroy; database = yield* createDatabase(); yield* startReady(database); diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index 8047ed1760..fabad493f8 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -51,6 +51,7 @@ import { } from "./services/Catalog.ts"; import * as State from "./State.ts"; import type { SavedInstance, SavedStack } from "./State.ts"; +import { makeDockerHelperRegistry } from "./storage/DockerHelperRegistry.ts"; export class OwnerError extends Data.TaggedError("OwnerError")<{ readonly operation: string; @@ -157,6 +158,7 @@ const makeOwnerWithDependencies = ( const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const http = yield* HttpClient.HttpClient; const ownerScope = yield* Scope.Scope; + const helpers = yield* makeDockerHelperRegistry; const recipes = yield* Ref.make(new Map()); const instances = yield* Ref.make(new Map>()); const namespaces = yield* Ref.make(new Map()); @@ -268,6 +270,7 @@ const makeOwnerWithDependencies = ( root: options.root, cacheRoot: options.cacheRoot, runtime, + helpers, }).pipe(Effect.mapError((cause) => errorFor("recipe", cause))); const recipeReady = (input: unknown, id: string) => recipeFor(input, id).pipe( diff --git a/packages/stack/src/services/Catalog.ts b/packages/stack/src/services/Catalog.ts index d300e627d3..678b5ac296 100644 --- a/packages/stack/src/services/Catalog.ts +++ b/packages/stack/src/services/Catalog.ts @@ -273,6 +273,7 @@ export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( root: options.root, cacheRoot: options.cacheRoot, runtime: options.runtime, + ...(options.helpers === undefined ? {} : { helpers: options.helpers }), }).pipe( Effect.mapError( (cause) => diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index 6c882e6288..7c19273893 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -51,6 +51,7 @@ import { import type { StackId } from "../identity/StackId.ts"; import { EndpointIntent, serviceCreation } from "./Recipe.ts"; import { makeDatabaseSnapshots } from "./DatabaseSnapshot.ts"; +import type { DockerHelperRegistry } from "../storage/DockerHelperRegistry.ts"; import { makeDockerDatabaseStorage, type DockerDatabaseStorage, @@ -106,6 +107,8 @@ export interface DatabaseOptions { readonly root: string; readonly cacheRoot: string; readonly runtime: DatabaseRuntime; + /** Reuses one volume helper across databases in this host. */ + readonly helpers?: DockerHelperRegistry; } export interface DatabaseComponent { @@ -134,6 +137,19 @@ const errorFor = (operation: string, cause: unknown): ServiceError => cause, }); +const postgresArguments = (config: DatabaseConfig): Array => { + const configured = new Set(Object.keys(config.settings ?? {}).map((key) => key.toLowerCase())); + const settings = Object.entries(config.settings ?? {}).flatMap(([key, value]) => [ + "-c", + `${key}=${String(value)}`, + ]); + // stopGraceSeconds 0 is the discarded shadow. Crash safety only adds wait. + if (config.stopGraceSeconds === 0) + for (const setting of ["fsync=off", "synchronous_commit=off", "full_page_writes=off"]) + if (!configured.has(setting.slice(0, setting.indexOf("=")))) settings.push("-c", setting); + return settings; +}; + const databaseError = (operation: string, cause: unknown): DatabaseError => new DatabaseError({ operation, @@ -168,6 +184,7 @@ const reconcileContainerPassword = Effect.fn("Database.reconcileContainerPasswor ) => Effect.scoped( Effect.gen(function* () { + yield* shadowPhase("reconcile-attempt"); const child = yield* spawner.spawn( ChildProcess.make( engine, @@ -236,12 +253,16 @@ const health = Effect.fn("Database.health")(( }), ); const client = Context.get(layer, PgClient.PgClient); + yield* shadowPhase("tcp-probe-attempt"); yield* client.unsafe("SELECT 1"); }), ); const retryProbe = probe.pipe(Effect.retry(Schedule.spaced("250 millis"))); return shadowPhase("health-probe-begin").pipe( + Effect.andThen(shadowPhase("reconcile-begin")), Effect.andThen(reconcile), + Effect.andThen(shadowPhase("reconcile-end")), + Effect.andThen(shadowPhase("tcp-probe-begin")), Effect.andThen(retryProbe), Effect.tap(() => shadowPhase("health-probe-ok")), Effect.andThen( @@ -509,6 +530,7 @@ export const makeDatabase = ( crypto, container, spawner, + ...(options.helpers === undefined ? {} : { helpers: options.helpers }), }).pipe(Effect.mapError((cause) => databaseError("storage", cause))); const snapshots = (version: string) => @@ -675,6 +697,100 @@ export const makeDatabase = ( Effect.mapError((cause) => errorFor("prepare", cause)), ); + const openDatabaseContainer = Effect.fn("Database.openContainer")(function* ( + context: ServiceInstanceContext, + ) { + const config = { ...context.config, version: postgresVersion(context.config.version) }; + const dataPath = path.join(instanceRoot, "data"); + const dataMount = + storage === undefined + ? yield* fs.makeDirectory(dataPath, { recursive: true, mode: 0o700 }).pipe( + Effect.mapError((cause) => errorFor("launch", cause)), + Effect.as({ + source: dataPath, + target: "/var/lib/postgresql/data", + readOnly: false, + }), + ) + : yield* storage + .mount(config.version) + .pipe(Effect.mapError((cause) => errorFor("launch", cause))); + const rootKeyPath = + config.rootKey === undefined ? undefined : path.join(instanceRoot, "pgsodium_root.key"); + if (rootKeyPath !== undefined && config.rootKey !== undefined) + yield* fs + .writeFileString(rootKeyPath, Redacted.value(config.rootKey), { + mode: options.runtime === "native" ? 0o600 : 0o644, + }) + .pipe(Effect.mapError((cause) => errorFor("launch", cause))); + const settings = postgresArguments(config); + const selectedContainer = container; + if (selectedContainer === undefined) + return yield* errorFor("launch", "Container runtime is unavailable"); + yield* shadowPhase("launch-image-begin"); + const image = yield* resolveArtifact({ + service: "database", + version: config.version, + }).pipe(Effect.mapError((cause) => errorFor("launch", cause))); + yield* shadowPhase("launch-image-end"); + if ( + storage === undefined || + (yield* storage.needsDataChown.pipe(Effect.mapError((cause) => errorFor("launch", cause)))) + ) { + yield* shadowPhase("launch-chown-begin"); + yield* dataCommand(config.version, ["chown", "100:101", "/var/lib/postgresql/data"]); + yield* shadowPhase("launch-chown-end"); + } + yield* shadowPhase("launch-container-begin"); + const launched = selectedContainer + .launch({ + image: image.image, + stackId: String(options.stackId), + instanceId: options.instanceId, + env: { + PGDATA: "/var/lib/postgresql/data", + PGSODIUM_KEY_FILE: + rootKeyPath === undefined + ? "/var/lib/postgresql/data/pgsodium_root.key" + : "/etc/postgresql-custom/pgsodium_root.key", + POSTGRES_USER: "supabase_admin", + POSTGRES_DB: "postgres", + POSTGRES_PASSWORD: Redacted.value(config.databasePassword), + }, + args: ["-p", "5432", "-c", "listen_addresses=*", ...settings], + mounts: [ + dataMount, + ...(rootKeyPath === undefined + ? [] + : [ + { + source: rootKeyPath, + target: "/etc/postgresql-custom/pgsodium_root.key", + readOnly: true, + }, + ]), + ], + ports: [5432], + ...(config.stopGraceSeconds === undefined + ? {} + : { stopGraceSeconds: config.stopGraceSeconds }), + }) + .pipe( + Effect.catchTag("ContainerLaunchError", ({ failure, process }) => + Effect.fail( + new ServiceLaunchError({ + failure: errorFor("launch", failure), + runtime: runtimeFromContainer(process, false), + }), + ), + ), + Effect.mapError((cause) => + cause instanceof ServiceLaunchError ? cause : errorFor("launch", cause), + ), + ); + return yield* launched.pipe(Scope.provide(context.scope)); + }); + const launch = Effect.fn("Database.launch")( ( context: ServiceInstanceContext, @@ -682,19 +798,10 @@ export const makeDatabase = ( Effect.gen(function* () { const config = { ...context.config, version: postgresVersion(context.config.version) }; const dataPath = path.join(instanceRoot, "data"); - const dataMount = - storage === undefined - ? yield* fs.makeDirectory(dataPath, { recursive: true, mode: 0o700 }).pipe( - Effect.mapError((cause) => errorFor("launch", cause)), - Effect.as({ - source: dataPath, - target: "/var/lib/postgresql/data", - readOnly: false, - }), - ) - : yield* storage - .mount(config.version) - .pipe(Effect.mapError((cause) => errorFor("launch", cause))); + if (storage === undefined) + yield* fs + .makeDirectory(dataPath, { recursive: true, mode: 0o700 }) + .pipe(Effect.mapError((cause) => errorFor("launch", cause))); const rootKeyPath = config.rootKey === undefined ? undefined : path.join(instanceRoot, "pgsodium_root.key"); if (rootKeyPath !== undefined && config.rootKey !== undefined) @@ -703,10 +810,7 @@ export const makeDatabase = ( mode: options.runtime === "native" ? 0o600 : 0o644, }) .pipe(Effect.mapError((cause) => errorFor("launch", cause))); - const settings = Object.entries(config.settings ?? {}).flatMap(([key, value]) => [ - "-c", - `${key}=${String(value)}`, - ]); + const settings = postgresArguments(config); if (options.runtime === "native") { // PostgreSQL limits Unix socket paths to 103 bytes, independently of the user's state root. const socketPath = yield* Effect.acquireRelease( @@ -762,73 +866,7 @@ export const makeDatabase = ( ), } satisfies RuntimeSession; } - yield* shadowPhase("launch-image-begin"); - const image = yield* resolveArtifact({ - service: "database", - version: config.version, - }).pipe(Effect.mapError((cause) => errorFor("launch", cause))); - yield* shadowPhase("launch-image-end"); - const selectedContainer = container; - if (selectedContainer === undefined) - return yield* errorFor("launch", "Container runtime is unavailable"); - if ( - storage === undefined || - (yield* storage.needsDataChown.pipe( - Effect.mapError((cause) => errorFor("launch", cause)), - )) - ) { - yield* shadowPhase("launch-chown-begin"); - yield* dataCommand(config.version, ["chown", "100:101", "/var/lib/postgresql/data"]); - yield* shadowPhase("launch-chown-end"); - } - yield* shadowPhase("launch-container-begin"); - const launched = yield* selectedContainer - .launch({ - image: image.image, - stackId: String(options.stackId), - instanceId: options.instanceId, - env: { - PGDATA: "/var/lib/postgresql/data", - PGSODIUM_KEY_FILE: - rootKeyPath === undefined - ? "/var/lib/postgresql/data/pgsodium_root.key" - : "/etc/postgresql-custom/pgsodium_root.key", - POSTGRES_USER: "supabase_admin", - POSTGRES_DB: "postgres", - POSTGRES_PASSWORD: Redacted.value(config.databasePassword), - }, - args: ["-p", "5432", "-c", "listen_addresses=*", ...settings], - mounts: [ - dataMount, - ...(rootKeyPath === undefined - ? [] - : [ - { - source: rootKeyPath, - target: "/etc/postgresql-custom/pgsodium_root.key", - readOnly: true, - }, - ]), - ], - ports: [5432], - ...(config.stopGraceSeconds === undefined - ? {} - : { stopGraceSeconds: config.stopGraceSeconds }), - }) - .pipe( - Effect.catchTag("ContainerLaunchError", ({ failure, process }) => - Effect.fail( - new ServiceLaunchError({ - failure: errorFor("launch", failure), - runtime: runtimeFromContainer(process, false), - }), - ), - ), - Effect.mapError((cause) => - cause instanceof ServiceLaunchError ? cause : errorFor("launch", cause), - ), - Scope.provide(context.scope), - ); + const launched = yield* openDatabaseContainer(context); yield* shadowPhase("launch-container-end"); const port = launched.ports[5432]; if (port === undefined) diff --git a/packages/stack/src/services/Recipe.ts b/packages/stack/src/services/Recipe.ts index 6e900c71c5..93cc5df0e6 100644 --- a/packages/stack/src/services/Recipe.ts +++ b/packages/stack/src/services/Recipe.ts @@ -3,6 +3,7 @@ import type { Stream } from "effect"; import type { Effect, Ref } from "effect"; import type { ServiceKind } from "../Artifacts.ts"; import type { ServiceDefinition } from "../Service.ts"; +import type { DockerHelperRegistry } from "../storage/DockerHelperRegistry.ts"; type CatalogRuntime = "native" | "docker" | "podman"; @@ -48,6 +49,8 @@ export interface CatalogOptions { readonly cacheRoot: string; readonly runtime: CatalogRuntime; readonly platform?: { readonly os: string; readonly arch: string }; + /** Reuses one volume helper across databases in this host. */ + readonly helpers?: DockerHelperRegistry; } export interface CatalogLog { diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index c426aae589..f979c377b2 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -16,6 +16,7 @@ import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/u import type { ContainerRuntime } from "../runtime/Container.ts"; import type { DatabaseRuntime } from "../services/Database.ts"; import { shadowPhase } from "../services/shadowPhase.ts"; +import type { DockerHelperRegistry } from "./DockerHelperRegistry.ts"; const HELPER_IMAGE = "public.ecr.aws/docker/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; @@ -52,6 +53,21 @@ const ReadyMarker = Schema.Struct({ runtime: Schema.Literals(["native", "docker", "podman"]), profile: Schema.Literal("supabase"), }); +const DaemonIdentity = Schema.Struct({ + daemonId: Schema.String, + clientMajor: Schema.Finite, + serverMajor: Schema.Finite, +}); +type DaemonIdentity = Schema.Schema.Type; +interface ResolvedDaemon extends DaemonIdentity { + readonly stateDigest: string; + readonly volume: string; + readonly cacheDigest: string; + /** True when this process queried the daemon, rather than reading the cache file. */ + readonly observed: boolean; + /** True when the shared volume was already present on this daemon. */ + readonly volumeConfirmed: boolean; +} export class DockerDatabaseStorageError extends Schema.TaggedError()( "DockerDatabaseStorageError", @@ -115,6 +131,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") readonly crypto: Crypto.Crypto; readonly container: ContainerRuntime | undefined; readonly spawner: ChildProcessSpawnerService["Service"]; + readonly helpers?: DockerHelperRegistry; }): Effect.Effect => Effect.gen(function* () { const markerPath = options.path.join(options.instanceRoot, ".supabase-database-storage.json"); @@ -140,8 +157,13 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/u.test(marker.volume)) ) return yield* errorFor("data", "Recorded database storage identity is invalid"); - return marker; + return marker; }); + // engineCommand is created just below; callers run only after this assignment. + let resolveDaemon: ( + forceLive: boolean, + ) => Effect.Effect = () => + Effect.die("Docker daemon identity was resolved before the engine command existed"); const selectedCache = yield* Ref.make(undefined); const selectionLock = yield* Semaphore.make(1); const selected = selectionLock.withPermit( @@ -203,29 +225,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") .pipe(Effect.mapError((cause) => errorFor("marker", cause))); return value; } - const daemonId = yield* engineCommand(["info", "--format", "{{.ID}}"]); - const version = yield* engineCommand([ - "version", - "--format", - "{{.Client.Version}}|{{.Server.Version}}", - ]); - const [clientVersion, serverVersion] = version.split("|"); - const clientMajor = parseMajor(clientVersion ?? ""); - const serverMajor = parseMajor(serverVersion ?? ""); - if (clientMajor === undefined || serverMajor === undefined) - return yield* errorFor("engine", "Docker returned an invalid version"); - const canonicalStateRoot = yield* options.fs - .realPath(stateRoot) - .pipe(Effect.mapError((cause) => errorFor("identity", cause))); - yield* options.fs - .makeDirectory(options.cacheRoot, { recursive: true }) - .pipe(Effect.mapError((cause) => errorFor("identity", cause))); - const canonicalCacheRoot = yield* options.fs - .realPath(options.cacheRoot) - .pipe(Effect.mapError((cause) => errorFor("identity", cause))); - const stateDigest = yield* hash(`${canonicalStateRoot}\0${daemonId.trim()}`); - const cacheDigest = yield* hash(canonicalCacheRoot); - const volume = `supabase-db-${stateDigest.slice(0, 32)}`; + let resolved = yield* resolveDaemon(false); const markerPresent = yield* options.fs .exists(markerPath) .pipe(Effect.mapError((cause) => errorFor("marker", cause))); @@ -240,38 +240,46 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") if (Option.isSome(marker)) { const validMarker = yield* validateMarker(marker.value); if (marker.value.backend === "docker") { - if (validMarker.daemonId !== daemonId.trim()) + if ( + !resolved.observed && + (validMarker.daemonId !== resolved.daemonId || + validMarker.volume !== resolved.volume) + ) + resolved = yield* resolveDaemon(true); + if (validMarker.daemonId !== resolved.daemonId) return yield* errorFor( "data", "Recorded Docker database storage does not match this daemon", ); if (validMarker.volume === undefined) return yield* errorFor("data", "Recorded Docker storage volume is missing"); - if (validMarker.volume !== volume) + if (validMarker.volume !== resolved.volume) return yield* errorFor( "data", "Recorded Docker database storage belongs to another state directory", ); - yield* engineCommand(["volume", "inspect", volume]).pipe( - Effect.catchTag("DockerDatabaseStorageError", (cause) => - !validMarker.initialized && /(?:no such volume|not found)/iu.test(cause.message) - ? engineCommand([ - "volume", - "create", - "--label", - "com.supabase.stack-managed=true", - "--label", - `com.supabase.stack-state-root=${stateDigest}`, - volume, - ]).pipe(Effect.asVoid) - : Effect.fail(cause), - ), - ); + if (!resolved.volumeConfirmed) + yield* engineCommand(["volume", "inspect", resolved.volume]).pipe( + Effect.catchTag("DockerDatabaseStorageError", (cause) => + !validMarker.initialized && + /(?:no such volume|not found)/iu.test(cause.message) + ? engineCommand([ + "volume", + "create", + "--label", + "com.supabase.stack-managed=true", + "--label", + `com.supabase.stack-state-root=${resolved.stateDigest}`, + resolved.volume, + ]).pipe(Effect.asVoid) + : Effect.fail(cause), + ), + ); } const updated = - validMarker.cacheNamespace === `cache-${cacheDigest.slice(0, 32)}` + validMarker.cacheNamespace === `cache-${resolved.cacheDigest.slice(0, 32)}` ? validMarker - : { ...validMarker, cacheNamespace: `cache-${cacheDigest.slice(0, 32)}` }; + : { ...validMarker, cacheNamespace: `cache-${resolved.cacheDigest.slice(0, 32)}` }; if (updated !== validMarker) yield* options.fs .writeFileString(markerPath, yield* encodeMarker(updated), { mode: 0o600 }) @@ -293,7 +301,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") const value: Marker = { backend: "host", namespace: dataNamespace, - cacheNamespace: `cache-${cacheDigest.slice(0, 32)}`, + cacheNamespace: `cache-${resolved.cacheDigest.slice(0, 32)}`, initialized: true, }; const encoded = yield* encodeMarker(value); @@ -304,24 +312,38 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") } } // Docker 26 introduced volume-subpath. Older engines retain the host-backed path. - const backend = clientMajor >= 26 && serverMajor >= 26 ? "docker" : "host"; + const backend = + resolved.clientMajor >= 26 && resolved.serverMajor >= 26 ? "docker" : "host"; const value: Marker = { backend, - ...(backend === "docker" ? { volume, daemonId: daemonId.trim() } : {}), + ...(backend === "docker" + ? { + volume: resolved.volume, + daemonId: resolved.daemonId, + } + : {}), namespace: dataNamespace, - cacheNamespace: `cache-${cacheDigest.slice(0, 32)}`, + cacheNamespace: `cache-${resolved.cacheDigest.slice(0, 32)}`, initialized: false, }; - if (backend === "docker") - yield* engineCommand([ - "volume", - "create", - "--label", - "com.supabase.stack-managed=true", - "--label", - `com.supabase.stack-state-root=${stateDigest}`, - volume, - ]); + if (backend === "docker" && !resolved.volumeConfirmed) { + yield* engineCommand(["volume", "inspect", resolved.volume]).pipe( + Effect.catchTag("DockerDatabaseStorageError", (cause) => + /no such volume|not found/iu.test(cause.message) + ? engineCommand([ + "volume", + "create", + "--label", + "com.supabase.stack-managed=true", + "--label", + `com.supabase.stack-state-root=${resolved.stateDigest}`, + resolved.volume, + ]).pipe(Effect.asVoid) + : Effect.fail(cause), + ), + Effect.asVoid, + ); + } yield* options.fs .writeFileString(markerPath, yield* encodeMarker(value), { mode: 0o600 }) .pipe(Effect.mapError((cause) => errorFor("marker", cause))); @@ -333,47 +355,143 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ); const engineCommand = Effect.fn("DockerDatabaseStorage.engineCommand")( - (args: ReadonlyArray): Effect.Effect => - Effect.scoped( - Effect.gen(function* () { - const child = yield* options.spawner - .spawn(ChildProcess.make(options.runtime, args, { stdin: "ignore" })) - .pipe(Effect.mapError((cause) => errorFor("engine", cause))); - const [stdout, stderr, code] = yield* Effect.all( - [ - child.stdout.pipe( - Stream.decodeText, - Stream.runFold( - () => "", - (all: string, chunk: string) => all + chunk, - ), - ), - child.stderr.pipe( - Stream.decodeText, - Stream.runFold( - () => "", - (all: string, chunk: string) => all + chunk, - ), - ), - child.exitCode.pipe(Effect.mapError((cause) => errorFor("engine", cause))), - ], - { concurrency: "unbounded" }, - ); - if (Number(code) !== 0) - return yield* errorFor( - "engine", - stderr.trim() || `Container engine exited with ${code}`, - ); - return stdout.trim(); - }), - ).pipe(Effect.mapError((cause) => errorFor("engine", cause))), + (args: ReadonlyArray): Effect.Effect => { + const command = args[0] ?? "command"; + return shadowPhase(`store-${command}-begin`).pipe( + Effect.andThen( + Effect.scoped( + Effect.gen(function* () { + const child = yield* options.spawner + .spawn(ChildProcess.make(options.runtime, args, { stdin: "ignore" })) + .pipe(Effect.mapError((cause) => errorFor("engine", cause))); + const [stdout, stderr, code] = yield* Effect.all( + [ + child.stdout.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (all: string, chunk: string) => all + chunk, + ), + ), + child.stderr.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (all: string, chunk: string) => all + chunk, + ), + ), + child.exitCode.pipe(Effect.mapError((cause) => errorFor("engine", cause))), + ], + { concurrency: "unbounded" }, + ); + if (Number(code) !== 0) + return yield* errorFor( + "engine", + stderr.trim() || `Container engine exited with ${code}`, + ); + return stdout.trim(); + }), + ).pipe(Effect.mapError((cause) => errorFor("engine", cause))), + ), + Effect.ensuring(shadowPhase(`store-${command}-end`)), + ); + }, ); - const validateDockerMarkerIdentity = (marker: Marker) => + const daemonIdentityRef = yield* Ref.make(undefined); + const encodeDaemonIdentity = Schema.encodeEffect(Schema.fromJsonString(DaemonIdentity)); + const identityPath = options.path.join(options.cacheRoot, "docker-daemon-identity.json"); + const volumePresent = (name: string) => + engineCommand(["volume", "inspect", name]).pipe( + Effect.as(true), + Effect.catchTag("DockerDatabaseStorageError", (cause) => + /no such volume|not found/iu.test(cause.message) + ? Effect.succeed(false) + : Effect.fail(cause), + ), + ); + const probeDaemonIdentity: Effect.Effect = Effect.gen(function* () { - const daemonId = yield* engineCommand(["info", "--format", "{{.ID}}"]).pipe( - Effect.map((value) => value.trim()), + const daemonId = (yield* engineCommand(["info", "--format", "{{.ID}}"])).trim(); + const version = yield* engineCommand([ + "version", + "--format", + "{{.Client.Version}}|{{.Server.Version}}", + ]); + const [clientVersion, serverVersion] = version.split("|"); + const clientMajor = parseMajor(clientVersion ?? ""); + const serverMajor = parseMajor(serverVersion ?? ""); + if (daemonId.length === 0 || clientMajor === undefined || serverMajor === undefined) + return yield* errorFor("engine", "Docker returned an invalid version"); + const identity = { daemonId, clientMajor, serverMajor }; + // The next process, including a shadow diff, reuses this instead of asking the daemon. + yield* encodeDaemonIdentity(identity).pipe( + Effect.flatMap((encoded) => + options.fs.writeFileString(identityPath, encoded, { mode: 0o600 }), + ), + Effect.ignore, ); - if (marker.daemonId !== daemonId) + return identity; + }); + const readDaemonIdentity = options.fs.readFileString(identityPath).pipe( + Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(DaemonIdentity))), + Effect.orElseSucceed(() => undefined), + ); + resolveDaemon = (forceLive) => + Effect.gen(function* () { + if (!forceLive) { + const remembered = yield* Ref.get(daemonIdentityRef); + if (remembered !== undefined) return remembered; + } + const canonicalStateRoot = yield* options.fs + .realPath(stateRoot) + .pipe(Effect.mapError((cause) => errorFor("identity", cause))); + yield* options.fs + .makeDirectory(options.cacheRoot, { recursive: true }) + .pipe(Effect.mapError((cause) => errorFor("identity", cause))); + const canonicalCacheRoot = yield* options.fs + .realPath(options.cacheRoot) + .pipe(Effect.mapError((cause) => errorFor("identity", cause))); + const cacheDigest = yield* hash(canonicalCacheRoot); + const describe = ( + identity: DaemonIdentity, + observed: boolean, + volumeConfirmed: boolean, + ) => + Effect.map(hash(`${canonicalStateRoot}\0${identity.daemonId}`), (stateDigest) => ({ + ...identity, + stateDigest, + volume: `supabase-db-${stateDigest.slice(0, 32)}`, + cacheDigest, + observed, + volumeConfirmed, + })); + if (!forceLive) { + const cached = yield* readDaemonIdentity; + if ( + cached !== undefined && + cached.daemonId.length > 0 && + Number.isInteger(cached.clientMajor) && + Number.isInteger(cached.serverMajor) + ) { + const candidate = yield* describe(cached, false, false); + if (yield* volumePresent(candidate.volume)) { + const accepted = { ...candidate, volumeConfirmed: true }; + yield* Ref.set(daemonIdentityRef, accepted); + return accepted; + } + } + } + const resolved = yield* describe(yield* probeDaemonIdentity, true, false); + yield* Ref.set(daemonIdentityRef, resolved); + return resolved; + }); + const validateDockerMarkerIdentity = (marker: Marker) => + Effect.gen(function* () { + let resolved = yield* resolveDaemon(false); + const matches = (candidate: ResolvedDaemon) => + marker.daemonId === candidate.daemonId && marker.volume === candidate.volume; + if (!matches(resolved) && !resolved.observed) resolved = yield* resolveDaemon(true); + if (marker.daemonId !== resolved.daemonId) return yield* errorFor( "destroy", "Recorded Docker database storage belongs to another daemon; switch back to the original Docker context before destroying it", @@ -381,7 +499,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") const canonicalStateRoot = yield* options.fs .realPath(stateRoot) .pipe(Effect.mapError((cause) => errorFor("identity", cause))); - const stateDigest = yield* hash(`${canonicalStateRoot}\0${daemonId}`); + const stateDigest = yield* hash(`${canonicalStateRoot}\0${resolved.daemonId}`); const expectedVolume = `supabase-db-${stateDigest.slice(0, 32)}`; if (marker.volume !== expectedVolume) return yield* errorFor( @@ -485,10 +603,106 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") }), ); + const missingContainer = (message: string) => /no such (?:container|object)/iu.test(message); + const volumeHelperKey = (mounts: ReadonlyArray): string | undefined => { + if (options.helpers === undefined) return undefined; + if (mounts.length === 0 || mounts.some((mount) => (mount.type ?? "bind") !== "volume")) + return undefined; + return mounts + .map( + (mount) => + `${mount.source}\0${mount.target}\0${mount.volumeSubpath ?? ""}\0${mount.readOnly ? "ro" : "rw"}`, + ) + .sort() + .join("\n"); + }; + const closeSharedHelper = (id: string) => + engineCommand(["rm", "-f", id]).pipe( + Effect.catchTag("DockerDatabaseStorageError", (cause) => + missingContainer(cause.message) ? Effect.void : Effect.fail(cause), + ), + Effect.asVoid, + ); + // A shadow stack finds the host helper already running and must not remove it. + const helperCreated = { current: false }; + const closeOwnedHelper = (id: string) => + helperCreated.current ? closeSharedHelper(id) : Effect.void; + const openSharedHelper = (mounts: ReadonlyArray, key: string) => + Effect.gen(function* () { + const name = `supabase-db-helper-${(yield* hash(key)).slice(0, 32)}`; + const running = yield* engineCommand([ + "inspect", + "--format", + "{{.State.Running}}", + name, + ]).pipe( + Effect.map((value) => value.trim() === "true"), + Effect.catchTag("DockerDatabaseStorageError", (cause) => + missingContainer(cause.message) ? Effect.succeed(false) : Effect.fail(cause), + ), + ); + if (running) { + helperCreated.current = false; + return name; + } + helperCreated.current = true; + yield* closeSharedHelper(name); + if (options.container === undefined) + return yield* errorFor("helper", "Container runtime is unavailable"); + yield* shadowPhase("helper-prepare-begin"); + yield* options.container.prepare(HELPER_IMAGE); + yield* shadowPhase("helper-prepare-end"); + yield* shadowPhase("helper-run-begin"); + yield* engineCommand([ + "run", + "-d", + "--name", + name, + "--label", + "com.supabase.stack-managed=true", + "--label", + "com.supabase.stack-helper=volume", + "--label", + `com.supabase.stack=${options.stackId}`, + ...mountArgs(mounts), + HELPER_IMAGE, + "/bin/sh", + "-c", + "trap : TERM INT; while :; do sleep 3600; done", + ]).pipe( + Effect.catchTag("DockerDatabaseStorageError", (cause) => + /already in use/iu.test(cause.message) + ? engineCommand(["inspect", "--format", "{{.State.Running}}", name]).pipe( + Effect.flatMap((value) => + value.trim() === "true" ? Effect.succeed(name) : Effect.fail(cause), + ), + ) + : Effect.fail(cause), + ), + ); + yield* shadowPhase("helper-run-end"); + return name; + }); const runHelper = Effect.fn("DockerDatabaseStorage.helper")(( command: string, mounts: ReadonlyArray, ): Effect.Effect => { + const sharedKey = volumeHelperKey(mounts); + if (sharedKey !== undefined && options.helpers !== undefined) { + const helpers = options.helpers; + const exec = (id: string) => engineCommand(["exec", id, "/bin/sh", "-c", command]); + const open = openSharedHelper(mounts, sharedKey); + return helpers.use(sharedKey, open, closeOwnedHelper, exec).pipe( + Effect.catchTag("DockerDatabaseStorageError", (cause) => + /no such container|is not running/iu.test(cause.message) + ? helpers.drop(sharedKey).pipe( + Effect.andThen(helpers.use(sharedKey, open, closeOwnedHelper, exec)), + ) + : Effect.fail(cause), + ), + Effect.mapError((cause) => errorFor("helper", cause)), + ); + } const cleanupAfterFailure = Effect.gen(function* () { if ((yield* Ref.get(helperId)) === undefined) return; yield* Ref.set(helperCleanupPending, true); @@ -647,16 +861,10 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") Effect.gen(function* () { const marker = yield* getMarker; if (marker.backend === "docker") { + const versionFile = `/store/${marker.namespace}/data/PG_VERSION`; yield* runHelper( - `set -eu; if [ ! -f ${shellQuote(`/store/${marker.namespace}/data/PG_VERSION`)} ]; then echo 'Database readiness requires PG_VERSION' >&2; exit 1; fi; actual=$(cat ${shellQuote(`/store/${marker.namespace}/data/PG_VERSION`)}); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Database PostgreSQL major does not match requested version' >&2; exit 1; fi`, - [ - { - source: marker.volume ?? "", - target: "/store", - readOnly: false, - type: "volume", - }, - ], + `set -eu; if [ ! -f ${shellQuote(versionFile)} ]; then echo 'Database readiness requires PG_VERSION' >&2; exit 1; fi; actual=$(cat ${shellQuote(versionFile)}); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Database PostgreSQL major does not match requested version' >&2; exit 1; fi`, + snapshotPaths(marker).mounts, ); } yield* writeMarker({ ...marker, initialized: true }); diff --git a/packages/stack/src/storage/DockerHelperRegistry.ts b/packages/stack/src/storage/DockerHelperRegistry.ts new file mode 100644 index 0000000000..e36ebd68d1 --- /dev/null +++ b/packages/stack/src/storage/DockerHelperRegistry.ts @@ -0,0 +1,82 @@ +import { Effect, Ref, Scope, Semaphore } from "effect"; + +export interface DockerHelperRegistry { + /** Runs `body` with the helper for `key`, opening it once per host lifetime. */ + readonly use: ( + key: string, + open: Effect.Effect, + close: (id: string) => Effect.Effect, + body: (id: string) => Effect.Effect, + ) => Effect.Effect; + /** Forgets a dead helper so the next use starts another. */ + readonly drop: (key: string) => Effect.Effect; +} + +/** + * One sleeping container per volume mount. Mounts are fixed when the container is created, + * and every database in a state directory uses that same volume. + */ +export const makeDockerHelperRegistry: Effect.Effect = + Effect.gen(function* () { + const helpers = yield* Ref.make( + new Map< + string, + { readonly id: string; readonly close: (id: string) => Effect.Effect } + >(), + ); + const gate = yield* Semaphore.make(1); + const scope = yield* Scope.Scope; + const release = (id: string, close: (id: string) => Effect.Effect) => + close(id).pipe(Effect.catch((cause) => Effect.logError(cause))); + yield* Scope.addFinalizer( + scope, + gate.withPermits(1)( + Effect.gen(function* () { + const current = yield* Ref.getAndSet(helpers, new Map()); + for (const entry of current.values()) yield* release(entry.id, entry.close); + }), + ), + ); + const use = ( + key: string, + open: Effect.Effect, + close: (id: string) => Effect.Effect, + body: (id: string) => Effect.Effect, + ): Effect.Effect => + gate.withPermit( + Effect.gen(function* () { + const current = yield* Ref.get(helpers); + const existing = current.get(key); + const id = + existing?.id ?? + (yield* open.pipe( + Effect.tap((opened) => + Ref.update(helpers, (map) => { + const next = new Map(map); + next.set(key, { + id: opened, + close: (helperId) => + close(helperId).pipe(Effect.catch((cause) => Effect.logError(cause))), + }); + return next; + }), + ), + )); + return yield* body(id); + }), + ); + const drop = (key: string): Effect.Effect => + gate.withPermit( + Effect.gen(function* () { + const entry = (yield* Ref.get(helpers)).get(key); + if (entry === undefined) return; + yield* Ref.update(helpers, (map) => { + const next = new Map(map); + next.delete(key); + return next; + }); + yield* release(entry.id, entry.close); + }), + ); + return { use, drop }; + }); diff --git a/packages/stack/src/storage/DockerHelperRegistry.unit.test.ts b/packages/stack/src/storage/DockerHelperRegistry.unit.test.ts new file mode 100644 index 0000000000..b267ca0d39 --- /dev/null +++ b/packages/stack/src/storage/DockerHelperRegistry.unit.test.ts @@ -0,0 +1,44 @@ +import { expect, it } from "@effect/vitest"; +import { Effect, Exit, Ref } from "effect"; +import { makeDockerHelperRegistry } from "./DockerHelperRegistry.ts"; + +it.effect("opens a helper once and closes it with the host scope", () => + Effect.gen(function* () { + const opened = yield* Ref.make(0); + const closed = yield* Ref.make>([]); + const seen = yield* Ref.make>([]); + yield* Effect.scoped( + Effect.gen(function* () { + const registry = yield* makeDockerHelperRegistry; + const open = Ref.updateAndGet(opened, (count) => count + 1).pipe( + Effect.map((count) => `helper-${String(count)}`), + ); + const close = (id: string) => Ref.update(closed, (ids) => [...ids, id]); + yield* registry.use("volume", open, close, (id) => Ref.update(seen, (ids) => [...ids, id])); + yield* registry.use("volume", open, close, (id) => Ref.update(seen, (ids) => [...ids, id])); + expect(yield* Ref.get(opened)).toBe(1); + expect(yield* Ref.get(seen)).toEqual(["helper-1", "helper-1"]); + expect(yield* Ref.get(closed)).toEqual([]); + }), + ); + expect(yield* Ref.get(closed)).toEqual(["helper-1"]); + }), +); + +it.effect("keeps the helper when a command fails", () => + Effect.gen(function* () { + const opened = yield* Ref.make(0); + const registry = yield* makeDockerHelperRegistry; + const open = Ref.updateAndGet(opened, (count) => count + 1).pipe( + Effect.map((count) => `helper-${String(count)}`), + ); + const close = (_id: string) => Effect.void; + const failed = yield* registry + .use("volume", open, close, () => Effect.fail("script")) + .pipe(Effect.exit); + expect(Exit.isFailure(failed)).toBe(true); + const id = yield* registry.use("volume", open, close, (value) => Effect.succeed(value)); + expect(id).toBe("helper-1"); + expect(yield* Ref.get(opened)).toBe(1); + }), +); From db440ea811d13428c90d72ac81a31303c5d88817 Mon Sep 17 00:00:00 2001 From: avallete Date: Tue, 22 Sep 2026 22:03:58 +0200 Subject: [PATCH 08/20] perf(stack): drop shadow timing probes The phase file was only for benchmarks. A shared helper is removed only by the process that created it, so another diff cannot delete the container the first one is using. --- apps/cli/src/command-internal/shadow-phase.ts | 13 -- apps/cli/src/command-internal/stack-shadow.ts | 36 +---- .../db/shared/pgdelta-engine.next.layer.ts | 10 +- packages/stack/src/Service.ts | 4 - packages/stack/src/runtime/Container.ts | 4 - packages/stack/src/services/Database.ts | 19 +-- packages/stack/src/services/shadowPhase.ts | 16 -- .../src/storage/DockerDatabaseStorage.ts | 149 +++++++++--------- .../stack/src/storage/DockerHelperRegistry.ts | 20 ++- .../storage/DockerHelperRegistry.unit.test.ts | 22 ++- 10 files changed, 112 insertions(+), 181 deletions(-) delete mode 100644 apps/cli/src/command-internal/shadow-phase.ts delete mode 100644 packages/stack/src/services/shadowPhase.ts diff --git a/apps/cli/src/command-internal/shadow-phase.ts b/apps/cli/src/command-internal/shadow-phase.ts deleted file mode 100644 index 8a658c934d..0000000000 --- a/apps/cli/src/command-internal/shadow-phase.ts +++ /dev/null @@ -1,13 +0,0 @@ -// oxlint-disable-next-line effecttsgo/node-builtin-import -- phase file is a CLI-side probe; FileSystem has no append. -import { appendFileSync } from "node:fs"; -import { Effect } from "effect"; - -/** Appends a timestamp when `SUPABASE_SHADOW_PHASE_FILE` is set. */ -export const shadowPhase = (label: string) => - Effect.suspend(() => { - const file = process.env["SUPABASE_SHADOW_PHASE_FILE"]; - if (file === undefined || file.length === 0) return Effect.void; - return Effect.try(() => { - appendFileSync(file, `${Date.now()} ${process.pid} ${label}\n`); - }).pipe(Effect.ignore); - }); diff --git a/apps/cli/src/command-internal/stack-shadow.ts b/apps/cli/src/command-internal/stack-shadow.ts index a5b5a35b4e..373e44000d 100644 --- a/apps/cli/src/command-internal/stack-shadow.ts +++ b/apps/cli/src/command-internal/stack-shadow.ts @@ -17,7 +17,6 @@ import { import type { SetupDatabaseOptions } from "./db-bootstrap/db-setup.ts"; import { listLocalMigrationPaths } from "./migration-history.ts"; import { applyMigrations } from "./migration-apply.ts"; -import { shadowPhase } from "./shadow-phase.ts"; import { stackShadowCacheEntry, stackShadowCacheRoles } from "./stack-shadow-cache.ts"; type Runtime = "native" | "docker" | "podman"; @@ -58,14 +57,12 @@ const acquireNamespace = Effect.fn("StackShadow.acquireNamespace")(function* (op const runtimeInfo = yield* RuntimeInfo; const runtime = opts.runtime ?? (yield* stackProjectRuntime) ?? defaultStackRuntime(runtimeInfo); const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-shadow-" }); - yield* shadowPhase("namespace-begin"); const stack = yield* api.create({ projectRoot: root, stateRoot: path.join(settings.supabaseHome, "stacks"), cacheRoot: path.join(settings.supabaseHome, "cache", "stack"), runtime, }); - yield* shadowPhase("namespace-end"); return { stack, runtime }; }); @@ -99,10 +96,7 @@ const initialize = Effect.fn("StackShadow.initialize")(function* ( }, endpoints: { sql: { port: opts.port ?? "auto" } }, }); - yield* shadowPhase("create-db-begin"); let database = yield* createDatabase(); - yield* shadowPhase("create-db-end"); - yield* shadowPhase("cache-begin"); const cacheResolution = yield* Effect.result( stackShadowCacheEntry( input, @@ -113,17 +107,12 @@ const initialize = Effect.fn("StackShadow.initialize")(function* ( opts.bypassCache, ), ); - yield* shadowPhase("cache-end"); const output = yield* Output; const cache = Result.isSuccess(cacheResolution) ? cacheResolution.success : undefined; const startReady = (db: DatabaseInstance) => Effect.gen(function* () { - yield* shadowPhase("start-begin"); yield* db.start; - yield* shadowPhase("start-end"); - yield* shadowPhase("ready-begin"); yield* db.ready; - yield* shadowPhase("ready-end"); }); if (Result.isFailure(cacheResolution)) yield* output.raw( @@ -134,9 +123,7 @@ const initialize = Effect.fn("StackShadow.initialize")(function* ( if (cache !== undefined) { const warmAttempt = yield* Effect.result( Effect.gen(function* () { - yield* shadowPhase("restore-begin"); const restoredSnapshot = yield* database.restoreSnapshot(cache.key); - yield* shadowPhase("restore-end"); if (!restoredSnapshot) return false; yield* startReady(database); return true; @@ -202,9 +189,7 @@ const initialize = Effect.fn("StackShadow.initialize")(function* ( } } } - yield* shadowPhase("credentials-begin"); const credentials = yield* database.credentials({ from: "host" }); - yield* shadowPhase("credentials-end"); const credentialUrl = credentials.databaseUrl; const credentialsConn = credentialUrl === undefined ? undefined : parseConnectionString(credentialUrl); @@ -235,18 +220,13 @@ export const stackAcquireShadowDatabase = Effect.fn("StackShadow.acquire")(funct ) { const output = yield* Output; const namespace = yield* Effect.acquireRelease(acquireNamespace(opts), ({ stack }) => - shadowPhase("destroy-begin").pipe( - Effect.andThen( - stack.destroy.pipe( - Effect.catch((cause) => - output.raw( - `Failed to destroy shadow stack ${stack.id}: ${cause.message}. Run supabase stack destroy --stack-id ${stack.id} to remove it.\n`, - "stderr", - ), - ), + stack.destroy.pipe( + Effect.catch((cause) => + output.raw( + `Failed to destroy shadow stack ${stack.id}: ${cause.message}. Run supabase stack destroy --stack-id ${stack.id} to remove it.\n`, + "stderr", ), ), - Effect.ensuring(shadowPhase("destroy-end")), ), ); return yield* initialize(namespace.stack, namespace.runtime, input, opts); @@ -265,7 +245,6 @@ export const stackMigrateShadow = Effect.fn("StackShadow.migrate")(function* ( handle: StackShadowAcquiredHandle, input: ShadowSetupInput, ) { - yield* shadowPhase("migrations-list-begin"); const migrationsDir = input.path.join(input.workdir, "supabase", "migrations"); const pending = yield* listLocalMigrationPaths(input.fs, input.path, migrationsDir).pipe( Effect.mapError((cause) => new ShadowDbError({ message: cause.message, reason: "filesystem" })), @@ -273,11 +252,7 @@ export const stackMigrateShadow = Effect.fn("StackShadow.migrate")(function* ( const conn = parseConnectionString(handle.url); if (conn === undefined) return yield* new ShadowDbError({ message: "Invalid shadow database URL", reason: "connect" }); - yield* shadowPhase("migrations-list-end"); - yield* shadowPhase("connect-begin"); const session = yield* connectShadowDatabase(conn); - yield* shadowPhase("connect-end"); - yield* shadowPhase("apply-begin"); yield* applyMigrations( session, input.fs, @@ -290,7 +265,6 @@ export const stackMigrateShadow = Effect.fn("StackShadow.migrate")(function* ( ), Effect.mapError(shadowError), ); - yield* shadowPhase("apply-end"); }); export const stackPrepareShadowSource = Effect.fn("StackShadow.prepareSource")( diff --git a/apps/cli/src/commands/db/shared/pgdelta-engine.next.layer.ts b/apps/cli/src/commands/db/shared/pgdelta-engine.next.layer.ts index b019a6dc57..58bcc00c36 100644 --- a/apps/cli/src/commands/db/shared/pgdelta-engine.next.layer.ts +++ b/apps/cli/src/commands/db/shared/pgdelta-engine.next.layer.ts @@ -36,7 +36,6 @@ import { reportPgDeltaNextDiagnostics, } from "./pgdelta-next-diagnostics.ts"; import { PgDeltaNextShadow } from "./pgdelta-next-shadow.service.ts"; -import { shadowPhase } from "../../../command-internal/shadow-phase.ts"; function pgDeltaNextConnectSuggestion(cause: unknown): string | undefined { if (cause instanceof DbConnectError) return cause.suggestion; @@ -292,16 +291,9 @@ export const pgDeltaNextEngineLayer = Layer.effect( diffDatabase: (input) => Effect.scoped( Effect.gen(function* () { - yield* shadowPhase("diff-source-begin"); const migrationsPool = yield* acquireDatabase(input.source, input.context.projectEnv); - yield* shadowPhase("diff-source-end"); - yield* shadowPhase("diff-target-begin"); const desiredPool = yield* acquireDatabase(input.target, input.context.projectEnv); - yield* shadowPhase("diff-target-end"); - yield* shadowPhase("diff-adapter-begin"); - const result = yield* diffPools(input, migrationsPool, desiredPool); - yield* shadowPhase("diff-adapter-end"); - return result; + return yield* diffPools(input, migrationsPool, desiredPool); }), ).pipe(Effect.mapError(pgDeltaNextEngineError)), exportDeclarativeSchema: (input) => diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index 87c56aafdd..389e75a5dc 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -12,7 +12,6 @@ import { SubscriptionRef, } from "effect"; import type { Stream } from "effect"; -import { shadowPhase } from "./services/shadowPhase.ts"; type ServiceLifecycle = "stopped" | "starting" | "running" | "stopping"; type ServiceHealth = "starting" | "healthy" | "unhealthy"; @@ -247,9 +246,6 @@ export const makeService = ( ); yield* Scope.close(record.healthScope, Exit.void); if (!(yield* Ref.get(record.stopped))) { - yield* shadowPhase( - `stopNow discard=${String(discard)} has=${String(record.runtime.discard !== undefined)}`, - ); const halt = discard && record.runtime.discard !== undefined ? record.runtime.discard diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index 3875071279..1f755cfb34 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -16,7 +16,6 @@ import { Stream, } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { shadowPhase } from "../services/shadowPhase.ts"; export class ContainerError extends Data.TaggedError("ContainerError")<{ readonly operation: string; @@ -257,7 +256,6 @@ export const makeContainerRuntime = (options: { const removed = yield* Ref.make(false); const stop = Effect.gen(function* () { if ((yield* Ref.get(removed)) || (yield* Ref.get(stopped))) return; - yield* shadowPhase("container-stop"); const grace = spec.stopGraceSeconds !== undefined && Number.isInteger(spec.stopGraceSeconds) && @@ -272,7 +270,6 @@ export const makeContainerRuntime = (options: { const beginStop = Effect.scoped( Effect.gen(function* () { if ((yield* Ref.get(removed)) || (yield* Ref.get(stopped))) return; - yield* shadowPhase("container-stop-background"); const script = `${options.engine} stop --time 10 ${id} && ${options.engine} rm ${id}`; const child = yield* Effect.mapError( spawner.spawn( @@ -299,7 +296,6 @@ export const makeContainerRuntime = (options: { ); const kill = Effect.gen(function* () { if ((yield* Ref.get(removed)) || (yield* Ref.get(stopped))) return; - yield* shadowPhase("container-kill"); yield* run(["kill", id]); yield* Ref.set(stopped, true); }); diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index 7c19273893..2f15618a88 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -25,7 +25,6 @@ import { resolveArtifact, type PreparedNativeArtifact, } from "../Artifacts.ts"; -import { shadowPhase } from "./shadowPhase.ts"; import { makeContainerRuntime, type ContainerProcess, @@ -184,7 +183,6 @@ const reconcileContainerPassword = Effect.fn("Database.reconcileContainerPasswor ) => Effect.scoped( Effect.gen(function* () { - yield* shadowPhase("reconcile-attempt"); const child = yield* spawner.spawn( ChildProcess.make( engine, @@ -253,18 +251,12 @@ const health = Effect.fn("Database.health")(( }), ); const client = Context.get(layer, PgClient.PgClient); - yield* shadowPhase("tcp-probe-attempt"); yield* client.unsafe("SELECT 1"); }), ); const retryProbe = probe.pipe(Effect.retry(Schedule.spaced("250 millis"))); - return shadowPhase("health-probe-begin").pipe( - Effect.andThen(shadowPhase("reconcile-begin")), - Effect.andThen(reconcile), - Effect.andThen(shadowPhase("reconcile-end")), - Effect.andThen(shadowPhase("tcp-probe-begin")), + return reconcile.pipe( Effect.andThen(retryProbe), - Effect.tap(() => shadowPhase("health-probe-ok")), Effect.andThen( Effect.scoped( Effect.gen(function* () { @@ -301,7 +293,6 @@ const health = Effect.fn("Database.health")(( }), ), ); - yield* shadowPhase("health-bootstrap-begin"); yield* ensureInternalDatabase(session, openInternal).pipe( Effect.mapError((cause) => errorFor("bootstrap", cause)), ); @@ -310,7 +301,6 @@ const health = Effect.fn("Database.health")(( jwtSecret: config.jwtSecret, jwtExpiry: config.jwtExpiry, }).pipe(Effect.mapError((cause) => errorFor("bootstrap", cause))); - yield* shadowPhase("health-bootstrap-end"); const readyMarker = yield* Schema.encodeEffect( Schema.fromJsonString(DatabaseReadyMarker), )({ @@ -727,21 +717,16 @@ export const makeDatabase = ( const selectedContainer = container; if (selectedContainer === undefined) return yield* errorFor("launch", "Container runtime is unavailable"); - yield* shadowPhase("launch-image-begin"); const image = yield* resolveArtifact({ service: "database", version: config.version, }).pipe(Effect.mapError((cause) => errorFor("launch", cause))); - yield* shadowPhase("launch-image-end"); if ( storage === undefined || (yield* storage.needsDataChown.pipe(Effect.mapError((cause) => errorFor("launch", cause)))) ) { - yield* shadowPhase("launch-chown-begin"); yield* dataCommand(config.version, ["chown", "100:101", "/var/lib/postgresql/data"]); - yield* shadowPhase("launch-chown-end"); } - yield* shadowPhase("launch-container-begin"); const launched = selectedContainer .launch({ image: image.image, @@ -867,14 +852,12 @@ export const makeDatabase = ( } satisfies RuntimeSession; } const launched = yield* openDatabaseContainer(context); - yield* shadowPhase("launch-container-end"); const port = launched.ports[5432]; if (port === undefined) return yield* errorFor("launch", "Container did not publish PostgreSQL"); const selectedEndpoint: BackendEndpoint = { kind: "tcp", host: "127.0.0.1", port }; yield* Ref.set(endpoint, selectedEndpoint); yield* publishLogs(launched, logs, context.scope); - yield* shadowPhase(`stop-grace-${String(config.stopGraceSeconds)}`); const session = runtimeFromContainer(launched, config.stopGraceSeconds === 0); return { ...session, diff --git a/packages/stack/src/services/shadowPhase.ts b/packages/stack/src/services/shadowPhase.ts deleted file mode 100644 index ea8b856980..0000000000 --- a/packages/stack/src/services/shadowPhase.ts +++ /dev/null @@ -1,16 +0,0 @@ -// oxlint-disable-next-line effecttsgo/node-builtin-import -- phase file is a host-side probe; FileSystem has no append. -import { appendFileSync } from "node:fs"; -import { Effect } from "effect"; - -/** Appends a timestamp when `SUPABASE_SHADOW_PHASE_FILE` is set. */ -export const shadowPhase = (label: string) => - Effect.suspend(() => { - // Config and Clock would add ConfigError to service methods that call this probe. - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- service-free probe, see above. - const file = process.env["SUPABASE_SHADOW_PHASE_FILE"]; - if (file === undefined || file.length === 0) return Effect.void; - return Effect.try(() => { - // oxlint-disable-next-line effecttsgo/global-date-in-effect -- service-free probe, see above. - appendFileSync(file, `${Date.now()} ${process.pid} ${label}\n`); - }).pipe(Effect.ignore); - }); diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index f979c377b2..0599c475a1 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -15,7 +15,6 @@ import { ChildProcess } from "effect/unstable/process"; import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; import type { ContainerRuntime } from "../runtime/Container.ts"; import type { DatabaseRuntime } from "../services/Database.ts"; -import { shadowPhase } from "../services/shadowPhase.ts"; import type { DockerHelperRegistry } from "./DockerHelperRegistry.ts"; const HELPER_IMAGE = @@ -157,7 +156,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/u.test(marker.volume)) ) return yield* errorFor("data", "Recorded database storage identity is invalid"); - return marker; + return marker; }); // engineCommand is created just below; callers run only after this assignment. let resolveDaemon: ( @@ -279,7 +278,10 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") const updated = validMarker.cacheNamespace === `cache-${resolved.cacheDigest.slice(0, 32)}` ? validMarker - : { ...validMarker, cacheNamespace: `cache-${resolved.cacheDigest.slice(0, 32)}` }; + : { + ...validMarker, + cacheNamespace: `cache-${resolved.cacheDigest.slice(0, 32)}`, + }; if (updated !== validMarker) yield* options.fs .writeFileString(markerPath, yield* encodeMarker(updated), { mode: 0o600 }) @@ -355,47 +357,40 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ); const engineCommand = Effect.fn("DockerDatabaseStorage.engineCommand")( - (args: ReadonlyArray): Effect.Effect => { - const command = args[0] ?? "command"; - return shadowPhase(`store-${command}-begin`).pipe( - Effect.andThen( - Effect.scoped( - Effect.gen(function* () { - const child = yield* options.spawner - .spawn(ChildProcess.make(options.runtime, args, { stdin: "ignore" })) - .pipe(Effect.mapError((cause) => errorFor("engine", cause))); - const [stdout, stderr, code] = yield* Effect.all( - [ - child.stdout.pipe( - Stream.decodeText, - Stream.runFold( - () => "", - (all: string, chunk: string) => all + chunk, - ), - ), - child.stderr.pipe( - Stream.decodeText, - Stream.runFold( - () => "", - (all: string, chunk: string) => all + chunk, - ), - ), - child.exitCode.pipe(Effect.mapError((cause) => errorFor("engine", cause))), - ], - { concurrency: "unbounded" }, - ); - if (Number(code) !== 0) - return yield* errorFor( - "engine", - stderr.trim() || `Container engine exited with ${code}`, - ); - return stdout.trim(); - }), - ).pipe(Effect.mapError((cause) => errorFor("engine", cause))), - ), - Effect.ensuring(shadowPhase(`store-${command}-end`)), - ); - }, + (args: ReadonlyArray): Effect.Effect => + Effect.scoped( + Effect.gen(function* () { + const child = yield* options.spawner + .spawn(ChildProcess.make(options.runtime, args, { stdin: "ignore" })) + .pipe(Effect.mapError((cause) => errorFor("engine", cause))); + const [stdout, stderr, code] = yield* Effect.all( + [ + child.stdout.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (all: string, chunk: string) => all + chunk, + ), + ), + child.stderr.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (all: string, chunk: string) => all + chunk, + ), + ), + child.exitCode.pipe(Effect.mapError((cause) => errorFor("engine", cause))), + ], + { concurrency: "unbounded" }, + ); + if (Number(code) !== 0) + return yield* errorFor( + "engine", + stderr.trim() || `Container engine exited with ${code}`, + ); + return stdout.trim(); + }), + ).pipe(Effect.mapError((cause) => errorFor("engine", cause))), ); const daemonIdentityRef = yield* Ref.make(undefined); const encodeDaemonIdentity = Schema.encodeEffect(Schema.fromJsonString(DaemonIdentity)); @@ -568,9 +563,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* options.fs .makeDirectory(options.cacheRoot, { recursive: true }) .pipe(Effect.mapError((cause) => errorFor("helper", cause))); - yield* shadowPhase("helper-prepare-begin"); yield* options.container.prepare(HELPER_IMAGE); - yield* shadowPhase("helper-prepare-end"); const token = yield* options.crypto.randomUUIDv4.pipe( Effect.mapError((cause) => errorFor("helper", cause)), ); @@ -578,7 +571,6 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") // Register the deterministic owned name before the remote create starts so an // interrupted docker run can still be removed by the same scope. yield* Ref.set(helperId, name); - yield* shadowPhase("helper-run-begin"); const created = yield* engineCommand([ "run", "-d", @@ -598,7 +590,6 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ]); if (!/^[a-f0-9]{12,64}$/u.test(created)) return yield* errorFor("helper", "Docker returned an invalid helper identity"); - yield* shadowPhase("helper-run-end"); return name; }), ); @@ -623,37 +614,46 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ), Effect.asVoid, ); - // A shadow stack finds the host helper already running and must not remove it. - const helperCreated = { current: false }; - const closeOwnedHelper = (id: string) => - helperCreated.current ? closeSharedHelper(id) : Effect.void; + // Only the process that created the container removes it. `docker rm` without -f + // fails while the container is running, so a racing creator cannot delete it. const openSharedHelper = (mounts: ReadonlyArray, key: string) => Effect.gen(function* () { const name = `supabase-db-helper-${(yield* hash(key)).slice(0, 32)}`; - const running = yield* engineCommand([ + const status = yield* engineCommand([ "inspect", "--format", - "{{.State.Running}}", + "{{.State.Status}}", name, ]).pipe( - Effect.map((value) => value.trim() === "true"), + Effect.map((value) => value.trim()), Effect.catchTag("DockerDatabaseStorageError", (cause) => - missingContainer(cause.message) ? Effect.succeed(false) : Effect.fail(cause), + missingContainer(cause.message) ? Effect.succeed("absent") : Effect.fail(cause), ), ); - if (running) { - helperCreated.current = false; - return name; + if ( + status === "running" || + status === "created" || + status === "restarting" || + status === "paused" + ) + return { id: name, created: false }; + if (status === "exited" || status === "dead") { + const removed = yield* engineCommand(["rm", name]).pipe( + Effect.map(() => "removed"), + Effect.catchTag("DockerDatabaseStorageError", (cause) => + missingContainer(cause.message) + ? Effect.succeed("absent") + : /is running|running container/iu.test(cause.message) + ? Effect.succeed("running") + : Effect.fail(cause), + ), + ); + if (removed === "running") return { id: name, created: false }; } - helperCreated.current = true; - yield* closeSharedHelper(name); if (options.container === undefined) return yield* errorFor("helper", "Container runtime is unavailable"); - yield* shadowPhase("helper-prepare-begin"); yield* options.container.prepare(HELPER_IMAGE); - yield* shadowPhase("helper-prepare-end"); - yield* shadowPhase("helper-run-begin"); - yield* engineCommand([ + return yield* engineCommand([ "run", "-d", "--name", @@ -670,18 +670,13 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") "-c", "trap : TERM INT; while :; do sleep 3600; done", ]).pipe( + Effect.as({ id: name, created: true }), Effect.catchTag("DockerDatabaseStorageError", (cause) => /already in use/iu.test(cause.message) - ? engineCommand(["inspect", "--format", "{{.State.Running}}", name]).pipe( - Effect.flatMap((value) => - value.trim() === "true" ? Effect.succeed(name) : Effect.fail(cause), - ), - ) + ? Effect.succeed({ id: name, created: false }) : Effect.fail(cause), ), ); - yield* shadowPhase("helper-run-end"); - return name; }); const runHelper = Effect.fn("DockerDatabaseStorage.helper")(( command: string, @@ -692,12 +687,12 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") const helpers = options.helpers; const exec = (id: string) => engineCommand(["exec", id, "/bin/sh", "-c", command]); const open = openSharedHelper(mounts, sharedKey); - return helpers.use(sharedKey, open, closeOwnedHelper, exec).pipe( + return helpers.use(sharedKey, open, closeSharedHelper, exec).pipe( Effect.catchTag("DockerDatabaseStorageError", (cause) => /no such container|is not running/iu.test(cause.message) - ? helpers.drop(sharedKey).pipe( - Effect.andThen(helpers.use(sharedKey, open, closeOwnedHelper, exec)), - ) + ? helpers + .drop(sharedKey) + .pipe(Effect.andThen(helpers.use(sharedKey, open, closeSharedHelper, exec))) : Effect.fail(cause), ), Effect.mapError((cause) => errorFor("helper", cause)), @@ -1043,12 +1038,10 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") store.backend === "host" ? `; owner=$(stat -c "%u:%g" /cache); mkdir -p /cache/stack-database-snapshots-helper; chown "$owner" /cache/stack-database-snapshots-helper; chown -R "$owner" ${root}` : ""; - yield* shadowPhase("restore-helper-begin"); const result = yield* runHelper( `set -eu; mkdir -p ${root}; flock -x -w 120 ${shellQuote(`${root}/.lock`)} sh -eu -c ${shellQuote(`set -eu; mkdir -p ${root}/entries ${root}/stages; find ${root}/stages -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +; find ${root}/entries -mindepth 1 -maxdepth 1 -name '*.retired' -exec rm -rf -- {} +; trap 'rm -rf ${stage}${cacheOwnership}' EXIT; if ! ${targetSetup}; then echo 'Initialized restore target data directory is missing' >&2; exit 1; fi; bad=$(find ${data} -mindepth 1 -print -quit); if [ -n "$bad" ]; then echo NONEMPTY; exit 0; fi; if [ ! -d ${source} ]; then echo MISS; exit 0; fi; if [ ! -f ${source}/descriptor.json ]; then echo 'Snapshot descriptor is missing' >&2; exit 1; fi; actual=$(cat ${source}/descriptor.json); expected=${shellQuote(descriptor)}; if [ "$actual" != "$expected" ]; then echo 'Snapshot descriptor does not match requested identity' >&2; exit 1; fi; bad=$(find ${source}/data \\( ! -type f ! -type d \\) -print -quit); if [ -n "$bad" ]; then echo "Snapshot contains unsupported filesystem entry $bad" >&2; exit 1; fi; if [ -e ${source}/data/postmaster.pid ]; then echo 'Snapshot contains postmaster.pid' >&2; exit 1; fi; rm -rf ${stage}; cp -a --reflink=auto ${source}/data ${stage}; actual=$(cat ${stage}/PG_VERSION); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Snapshot PostgreSQL major does not match requested version' >&2; exit 1; fi; ${store.backend === "host" ? `chown -R 100:101 ${stage};` : ""} rmdir ${data}; mv ${stage} ${data}; touch ${source}; echo HIT`)}; rm -rf ${shellQuote(stage)}`, paths.mounts, ); - yield* shadowPhase("restore-helper-end"); if (result === "MISS") return false; if (result === "NONEMPTY") return yield* errorFor("restore", "Restore target data directory must be empty"); diff --git a/packages/stack/src/storage/DockerHelperRegistry.ts b/packages/stack/src/storage/DockerHelperRegistry.ts index e36ebd68d1..a1386d6002 100644 --- a/packages/stack/src/storage/DockerHelperRegistry.ts +++ b/packages/stack/src/storage/DockerHelperRegistry.ts @@ -1,10 +1,16 @@ import { Effect, Ref, Scope, Semaphore } from "effect"; +export interface DockerHelperHandle { + readonly id: string; + /** False when this process found a helper another process already created. */ + readonly created: boolean; +} + export interface DockerHelperRegistry { /** Runs `body` with the helper for `key`, opening it once per host lifetime. */ readonly use: ( key: string, - open: Effect.Effect, + open: Effect.Effect, close: (id: string) => Effect.Effect, body: (id: string) => Effect.Effect, ) => Effect.Effect; @@ -26,8 +32,7 @@ export const makeDockerHelperRegistry: Effect.Effect Effect.Effect) => - close(id).pipe(Effect.catch((cause) => Effect.logError(cause))); + const release = (id: string, close: (id: string) => Effect.Effect) => close(id); yield* Scope.addFinalizer( scope, gate.withPermits(1)( @@ -39,7 +44,7 @@ export const makeDockerHelperRegistry: Effect.Effect( key: string, - open: Effect.Effect, + open: Effect.Effect, close: (id: string) => Effect.Effect, body: (id: string) => Effect.Effect, ): Effect.Effect => @@ -54,13 +59,16 @@ export const makeDockerHelperRegistry: Effect.Effect { const next = new Map(map); next.set(key, { - id: opened, + id: opened.id, close: (helperId) => - close(helperId).pipe(Effect.catch((cause) => Effect.logError(cause))), + (opened.created ? close(helperId) : Effect.void).pipe( + Effect.catch((cause) => Effect.logError(cause)), + ), }); return next; }), ), + Effect.map((opened) => opened.id), )); return yield* body(id); }), diff --git a/packages/stack/src/storage/DockerHelperRegistry.unit.test.ts b/packages/stack/src/storage/DockerHelperRegistry.unit.test.ts index b267ca0d39..ef18e6f66e 100644 --- a/packages/stack/src/storage/DockerHelperRegistry.unit.test.ts +++ b/packages/stack/src/storage/DockerHelperRegistry.unit.test.ts @@ -11,7 +11,7 @@ it.effect("opens a helper once and closes it with the host scope", () => Effect.gen(function* () { const registry = yield* makeDockerHelperRegistry; const open = Ref.updateAndGet(opened, (count) => count + 1).pipe( - Effect.map((count) => `helper-${String(count)}`), + Effect.map((count) => ({ id: `helper-${String(count)}`, created: true })), ); const close = (id: string) => Ref.update(closed, (ids) => [...ids, id]); yield* registry.use("volume", open, close, (id) => Ref.update(seen, (ids) => [...ids, id])); @@ -30,7 +30,7 @@ it.effect("keeps the helper when a command fails", () => const opened = yield* Ref.make(0); const registry = yield* makeDockerHelperRegistry; const open = Ref.updateAndGet(opened, (count) => count + 1).pipe( - Effect.map((count) => `helper-${String(count)}`), + Effect.map((count) => ({ id: `helper-${String(count)}`, created: true })), ); const close = (_id: string) => Effect.void; const failed = yield* registry @@ -42,3 +42,21 @@ it.effect("keeps the helper when a command fails", () => expect(yield* Ref.get(opened)).toBe(1); }), ); + +it.effect("does not remove a helper this process did not create", () => + Effect.gen(function* () { + const closed = yield* Ref.make>([]); + yield* Effect.scoped( + Effect.gen(function* () { + const registry = yield* makeDockerHelperRegistry; + yield* registry.use( + "volume", + Effect.succeed({ id: "adopted", created: false }), + (id) => Ref.update(closed, (ids) => [...ids, id]), + (id) => Effect.succeed(id), + ); + }), + ); + expect(yield* Ref.get(closed)).toEqual([]); + }), +); From dc135b13e8e487dcf52c1478fe30dd67a899a8ec Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 23 Sep 2026 09:46:17 +0200 Subject: [PATCH 09/20] fix(stack): recreate a helper that never started docker exec cannot use a container left in created, and adopting it again left later diffs stuck. The helper handle type stays private. --- packages/stack/src/storage/DockerDatabaseStorage.ts | 10 +++------- packages/stack/src/storage/DockerHelperRegistry.ts | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index 0599c475a1..6edde9b15a 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -616,6 +616,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ); // Only the process that created the container removes it. `docker rm` without -f // fails while the container is running, so a racing creator cannot delete it. + // A container left in `created` cannot exec; remove it and start another. const openSharedHelper = (mounts: ReadonlyArray, key: string) => Effect.gen(function* () { const name = `supabase-db-helper-${(yield* hash(key)).slice(0, 32)}`; @@ -630,14 +631,9 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") missingContainer(cause.message) ? Effect.succeed("absent") : Effect.fail(cause), ), ); - if ( - status === "running" || - status === "created" || - status === "restarting" || - status === "paused" - ) + if (status === "running" || status === "restarting" || status === "paused") return { id: name, created: false }; - if (status === "exited" || status === "dead") { + if (status === "created" || status === "exited" || status === "dead") { const removed = yield* engineCommand(["rm", name]).pipe( Effect.map(() => "removed"), Effect.catchTag("DockerDatabaseStorageError", (cause) => diff --git a/packages/stack/src/storage/DockerHelperRegistry.ts b/packages/stack/src/storage/DockerHelperRegistry.ts index a1386d6002..c3bf659cbb 100644 --- a/packages/stack/src/storage/DockerHelperRegistry.ts +++ b/packages/stack/src/storage/DockerHelperRegistry.ts @@ -1,6 +1,6 @@ import { Effect, Ref, Scope, Semaphore } from "effect"; -export interface DockerHelperHandle { +interface DockerHelperHandle { readonly id: string; /** False when this process found a helper another process already created. */ readonly created: boolean; From 9aab6a43f1e5b8ca725b269f981385f3ff962de0 Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 23 Sep 2026 10:51:26 +0200 Subject: [PATCH 10/20] fix(stack): finish partial snapshot copy cleanup after cancel A cancel during chmod left the failed host copy in place, and the interrupt test read the last argument, which is /NP on Windows. --- .../storage/DirectoryCopy.integration.test.ts | 129 ++++++++++++++++-- packages/stack/src/storage/DirectoryCopy.ts | 28 ++-- 2 files changed, 132 insertions(+), 25 deletions(-) diff --git a/packages/stack/src/storage/DirectoryCopy.integration.test.ts b/packages/stack/src/storage/DirectoryCopy.integration.test.ts index 64ed67766a..faa612af47 100644 --- a/packages/stack/src/storage/DirectoryCopy.integration.test.ts +++ b/packages/stack/src/storage/DirectoryCopy.integration.test.ts @@ -2,7 +2,18 @@ import { chmodSync, mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; -import { Deferred, Effect, Exit, Fiber, FileSystem, Path, Scope, Sink, Stream } from "effect"; +import { + Cause, + Deferred, + Effect, + Exit, + Fiber, + FileSystem, + Path, + Scope, + Sink, + Stream, +} from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { copyDirectory } from "./DirectoryCopy.ts"; @@ -148,6 +159,30 @@ describe("copyDirectory", () => { ), ); + it.live("removes a failed host copy and copies the tree itself", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "directory-copy-fallback-" }); + const source = path.join(root, "source"); + const destination = path.join(root, "destination"); + yield* fs.makeDirectory(source); + yield* fs.writeFileString(path.join(source, "file.txt"), "file\n"); + + yield* copyDirectory(source, destination).pipe( + Effect.provideService( + ChildProcessSpawner.ChildProcessSpawner, + failedHostCopy(destination), + ), + ); + + expect(yield* fs.readFileString(path.join(destination, "file.txt"))).toBe("file\n"); + expect(yield* fs.exists(path.join(destination, "nested"))).toBe(false); + }), + ), + ); + it.live("removes a partial copy when the host copy is interrupted", () => run( Effect.gen(function* () { @@ -161,7 +196,10 @@ describe("copyDirectory", () => { yield* fs.writeFileString(path.join(source, "file.txt"), "file\n"); const fiber = yield* copyDirectory(source, destination).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, hangingCopy(started)), + Effect.provideService( + ChildProcessSpawner.ChildProcessSpawner, + hangingCopy(started, destination), + ), Effect.forkChild, ); yield* Deferred.await(started); @@ -171,6 +209,52 @@ describe("copyDirectory", () => { }), ), ); + + it.live("removes a partial copy when failure cleanup is interrupted", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "directory-copy-cleanup-" }); + const source = path.join(root, "source"); + const destination = path.join(root, "destination"); + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + yield* fs.makeDirectory(source); + yield* fs.writeFileString(path.join(source, "file.txt"), "file\n"); + const pausing = FileSystem.FileSystem.of({ + ...fs, + chmod: (target, mode) => + Effect.gen(function* () { + if (target === destination) { + yield* Deferred.succeed(entered, undefined); + yield* Deferred.await(release); + } + yield* fs.chmod(target, mode); + }), + }); + + const fiber = yield* copyDirectory(source, destination).pipe( + Effect.provideService(FileSystem.FileSystem, pausing), + Effect.provideService( + ChildProcessSpawner.ChildProcessSpawner, + failedHostCopy(destination), + ), + Effect.forkChild, + ); + yield* Deferred.await(entered); + yield* Effect.sync(() => { + fiber.interruptUnsafe(); + }); + expect(fiber.pollUnsafe()).toBeUndefined(); + yield* Deferred.succeed(release, undefined); + const exit = yield* Fiber.await(fiber); + + expect(Exit.isFailure(exit) && Cause.hasInterrupts(exit.cause)).toBe(true); + expect(yield* fs.exists(destination)).toBe(false); + }), + ), + ); }); const processHandle = (exitCode: Effect.Effect) => @@ -188,19 +272,38 @@ const processHandle = (exitCode: Effect.Effect) => unref: Effect.succeed(Effect.void), }); -const hangingCopy = (started: Deferred.Deferred) => +const writePartial = (destination: string) => { + const nested = join(destination, "nested"); + mkdirSync(nested, { recursive: true }); + writeFileSync(join(nested, "child.txt"), "partial\n"); + if (process.platform !== "win32") chmodSync(nested, 0o555); +}; + +const hostCommand = (command: ChildProcess.Command) => { + if (!ChildProcess.isStandardCommand(command)) return Effect.die("unexpected piped command"); + if (command.command === "find") { + return Effect.succeed(processHandle(Effect.succeed(ChildProcessSpawner.ExitCode(0)))); + } + return undefined; +}; + +const hangingCopy = (started: Deferred.Deferred, destination: string) => ChildProcessSpawner.make((command) => { - if (!ChildProcess.isStandardCommand(command)) return Effect.die("unexpected piped command"); - if (command.command === "find") { - return Effect.succeed(processHandle(Effect.succeed(ChildProcessSpawner.ExitCode(0)))); - } - const destination = command.args.at(-1); - if (destination === undefined) return Effect.die("missing copy destination"); + const find = hostCommand(command); + if (find !== undefined) return find; return Effect.sync(() => { - const nested = join(destination, "nested"); - mkdirSync(nested, { recursive: true }); - writeFileSync(join(nested, "child.txt"), "partial\n"); - if (process.platform !== "win32") chmodSync(nested, 0o555); + writePartial(destination); return processHandle(Effect.never); }).pipe(Effect.tap(() => Deferred.succeed(started, undefined))); }); + +// Exit 8 is a real robocopy failure and a non-zero cp status, not a usage error. +const failedHostCopy = (destination: string) => + ChildProcessSpawner.make((command) => { + const find = hostCommand(command); + if (find !== undefined) return find; + return Effect.sync(() => { + writePartial(destination); + return processHandle(Effect.succeed(ChildProcessSpawner.ExitCode(8))); + }); + }); diff --git a/packages/stack/src/storage/DirectoryCopy.ts b/packages/stack/src/storage/DirectoryCopy.ts index d6f7c898ac..a5c2b4dd53 100644 --- a/packages/stack/src/storage/DirectoryCopy.ts +++ b/packages/stack/src/storage/DirectoryCopy.ts @@ -352,23 +352,27 @@ const grantDirectoryWrite = ( } }); +// Deletion has to finish after a cancel. Stopping during chmod leaves the partial tree, +// including directories whose mode then rejects removal. const removePartial = ( destination: string, fs: FileSystem.FileSystem, path: Path.Path, ): Effect.Effect => - Effect.gen(function* () { - if ( - !(yield* fs - .exists(destination) - .pipe(Effect.mapError((cause) => errorFor("remove", destination, destination, cause)))) - ) - return; - yield* grantDirectoryWrite(destination, fs, path); - yield* fs - .remove(destination, { recursive: true, force: true }) - .pipe(Effect.mapError((cause) => errorFor("remove", destination, destination, cause))); - }); + Effect.uninterruptible( + Effect.gen(function* () { + if ( + !(yield* fs + .exists(destination) + .pipe(Effect.mapError((cause) => errorFor("remove", destination, destination, cause)))) + ) + return; + yield* grantDirectoryWrite(destination, fs, path); + yield* fs + .remove(destination, { recursive: true, force: true }) + .pipe(Effect.mapError((cause) => errorFor("remove", destination, destination, cause))); + }), + ); const isUsageError = (cause: unknown): boolean => { if (typeof cause !== "object" || cause === null || !("message" in cause)) return false; From ec146da75062aa23882179c650f104c07d992ef1 Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 23 Sep 2026 11:25:07 +0200 Subject: [PATCH 11/20] test(stack): keep the interrupted copy fixture on Effect file APIs Effect lint rejects node:fs and node:path in this package, and those warnings fail the quality check. --- .../storage/DirectoryCopy.integration.test.ts | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/packages/stack/src/storage/DirectoryCopy.integration.test.ts b/packages/stack/src/storage/DirectoryCopy.integration.test.ts index 64ed67766a..a393f3a42d 100644 --- a/packages/stack/src/storage/DirectoryCopy.integration.test.ts +++ b/packages/stack/src/storage/DirectoryCopy.integration.test.ts @@ -1,5 +1,3 @@ -import { chmodSync, mkdirSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Deferred, Effect, Exit, Fiber, FileSystem, Path, Scope, Sink, Stream } from "effect"; @@ -161,7 +159,10 @@ describe("copyDirectory", () => { yield* fs.writeFileString(path.join(source, "file.txt"), "file\n"); const fiber = yield* copyDirectory(source, destination).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, hangingCopy(started)), + Effect.provideService( + ChildProcessSpawner.ChildProcessSpawner, + hangingCopy(started, fs, path), + ), Effect.forkChild, ); yield* Deferred.await(started); @@ -188,7 +189,11 @@ const processHandle = (exitCode: Effect.Effect) => unref: Effect.succeed(Effect.void), }); -const hangingCopy = (started: Deferred.Deferred) => +const hangingCopy = ( + started: Deferred.Deferred, + fs: FileSystem.FileSystem, + path: Path.Path, +) => ChildProcessSpawner.make((command) => { if (!ChildProcess.isStandardCommand(command)) return Effect.die("unexpected piped command"); if (command.command === "find") { @@ -196,11 +201,14 @@ const hangingCopy = (started: Deferred.Deferred) => } const destination = command.args.at(-1); if (destination === undefined) return Effect.die("missing copy destination"); - return Effect.sync(() => { - const nested = join(destination, "nested"); - mkdirSync(nested, { recursive: true }); - writeFileSync(join(nested, "child.txt"), "partial\n"); - if (process.platform !== "win32") chmodSync(nested, 0o555); + return Effect.gen(function* () { + const nested = path.join(destination, "nested"); + yield* fs.makeDirectory(nested, { recursive: true }); + yield* fs.writeFileString(path.join(nested, "child.txt"), "partial\n"); + if (process.platform !== "win32") yield* fs.chmod(nested, 0o555); return processHandle(Effect.never); - }).pipe(Effect.tap(() => Deferred.succeed(started, undefined))); + }).pipe( + Effect.orDie, + Effect.tap(() => Deferred.succeed(started, undefined)), + ); }); From 519f8f5ed7fabaa768eb9aa89d477e94e31db9e2 Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 23 Sep 2026 16:27:01 +0200 Subject: [PATCH 12/20] test(stack): give catalog services separate docker data Parallel tests reused one stack id, so on Linux they shared a data directory and the pooler database exited during ready. --- .../src/services/AnalyticsVectorImgproxy.integration.test.ts | 2 +- packages/stack/src/services/AuthStorage.integration.test.ts | 2 +- packages/stack/src/services/Catalog.integration.test.ts | 2 +- packages/stack/src/services/Functions.integration.test.ts | 2 +- packages/stack/src/services/Mail.integration.test.ts | 2 +- packages/stack/src/services/Pooler.integration.test.ts | 2 +- packages/stack/src/services/Realtime.integration.test.ts | 2 +- packages/stack/src/services/Rest.integration.test.ts | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts index f22ee4eb57..0a6cfabef0 100644 --- a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts +++ b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts @@ -8,7 +8,7 @@ import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-rela import { cleanupDockerRoot } from "../../tests/docker-cleanup.ts"; const options = (root: string) => ({ - stackId: "catalog-test", + stackId: "catalog-analytics", instanceId: "instance", root, cacheRoot: `${root}/cache`, diff --git a/packages/stack/src/services/AuthStorage.integration.test.ts b/packages/stack/src/services/AuthStorage.integration.test.ts index bc16f5a77a..4de695cf0f 100644 --- a/packages/stack/src/services/AuthStorage.integration.test.ts +++ b/packages/stack/src/services/AuthStorage.integration.test.ts @@ -9,7 +9,7 @@ import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-rela import { cleanupDockerRoot } from "../../tests/docker-cleanup.ts"; const options = (root: string) => ({ - stackId: "catalog-test", + stackId: "catalog-auth-storage", instanceId: "instance", root, cacheRoot: `${root}/cache`, diff --git a/packages/stack/src/services/Catalog.integration.test.ts b/packages/stack/src/services/Catalog.integration.test.ts index b423e54a18..be06f948a4 100644 --- a/packages/stack/src/services/Catalog.integration.test.ts +++ b/packages/stack/src/services/Catalog.integration.test.ts @@ -4,7 +4,7 @@ import { Effect, Exit, FileSystem, Layer, Redacted } from "effect"; import { makeServiceRecipe } from "./Catalog.ts"; const options = (root: string) => ({ - stackId: "catalog-test", + stackId: "catalog-recipe", instanceId: "instance", root, cacheRoot: `${root}/cache`, diff --git a/packages/stack/src/services/Functions.integration.test.ts b/packages/stack/src/services/Functions.integration.test.ts index 70d12dc7d9..43bdeec4e0 100644 --- a/packages/stack/src/services/Functions.integration.test.ts +++ b/packages/stack/src/services/Functions.integration.test.ts @@ -7,7 +7,7 @@ import { bundleServeMainTemplate } from "../../tests/serve-main-bundler.ts"; import { makeServiceRecipe } from "./Catalog.ts"; const options = (root: string) => ({ - stackId: "catalog-test", + stackId: "catalog-functions", instanceId: "instance", root, cacheRoot: `${root}/cache`, diff --git a/packages/stack/src/services/Mail.integration.test.ts b/packages/stack/src/services/Mail.integration.test.ts index 4727dae9c1..eea7be5eff 100644 --- a/packages/stack/src/services/Mail.integration.test.ts +++ b/packages/stack/src/services/Mail.integration.test.ts @@ -6,7 +6,7 @@ import { makeService } from "../Service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; const options = (root: string) => ({ - stackId: "catalog-test", + stackId: "catalog-mail", instanceId: "instance", root, cacheRoot: `${root}/cache`, diff --git a/packages/stack/src/services/Pooler.integration.test.ts b/packages/stack/src/services/Pooler.integration.test.ts index 8e5aa5e8de..61db6b079a 100644 --- a/packages/stack/src/services/Pooler.integration.test.ts +++ b/packages/stack/src/services/Pooler.integration.test.ts @@ -9,7 +9,7 @@ import { makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { cleanupDockerRoot } from "../../tests/docker-cleanup.ts"; const options = (root: string) => ({ - stackId: "catalog-test", + stackId: "catalog-pooler", instanceId: "instance", root, cacheRoot: `${root}/cache`, diff --git a/packages/stack/src/services/Realtime.integration.test.ts b/packages/stack/src/services/Realtime.integration.test.ts index 9b04f65925..ce6ddcc8c5 100644 --- a/packages/stack/src/services/Realtime.integration.test.ts +++ b/packages/stack/src/services/Realtime.integration.test.ts @@ -8,7 +8,7 @@ import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-rela import { cleanupDockerRoot } from "../../tests/docker-cleanup.ts"; const options = (root: string) => ({ - stackId: "catalog-test", + stackId: "catalog-realtime", instanceId: "instance", root, cacheRoot: `${root}/cache`, diff --git a/packages/stack/src/services/Rest.integration.test.ts b/packages/stack/src/services/Rest.integration.test.ts index cf72eae675..4775905f20 100644 --- a/packages/stack/src/services/Rest.integration.test.ts +++ b/packages/stack/src/services/Rest.integration.test.ts @@ -29,7 +29,7 @@ const makeTestNetwork = (options: { ).pipe(Effect.map((context) => Context.get(context, Network.Service))); const options = (root: string) => ({ - stackId: "catalog-test", + stackId: "catalog-rest", instanceId: "instance", root, cacheRoot: `${root}/cache`, From 39c15d60502e20bc2f585e40a102ce25102d481a Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 23 Sep 2026 17:54:16 +0200 Subject: [PATCH 13/20] perf(stack): reuse postgres image for storage helper --- packages/stack/src/Artifacts.ts | 4 +- .../DockerDatabaseStorage.integration.test.ts | 37 +++- .../src/storage/DockerDatabaseStorage.ts | 163 +++++++++++------- 3 files changed, 137 insertions(+), 67 deletions(-) diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index 1f7ed92735..97686a3c76 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -72,12 +72,12 @@ const definitions: Readonly> = { database: definition( "postgres", "17.6.1.173", - "ghcr.io/supabase/cli/postgres:17.6.1.173@sha256:1581c433d71a48a81e356a3ed2d4aa5ecfc8fc0465ea98661da7a88023317dcf", + "ghcr.io/supabase/cli/postgres:17.6.1.173@sha256:9d6e542382946cad5eb1f11f1c8108a51297902ee42fe5098358816d3784ba5a", "bin/supabase-postgres-start", ["bin/supabase-postgres-start", "bin/pg_dump", "bin/pg_dumpall", "bin/pg_prove", "bin/psql"], { "15.14.1.173": - "ghcr.io/supabase/cli/postgres:15.14.1.173@sha256:b7d210fa3bca26568fa20448e3bec092bd4f8da6a5af5029597639d38e7d896e", + "ghcr.io/supabase/cli/postgres:15.14.1.173@sha256:3abb20e89700d6211e741148b85c3a052bb290cae3b4f751311442f4a5fe2324", }, ), rest: definition("postgrest", "v16.2", "ghcr.io/supabase/cli/postgrest:v16.2", "bin/postgrest"), diff --git a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts index 697f150baa..5566992e29 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts @@ -13,12 +13,15 @@ import { Stream, } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { postgresVersion, resolveArtifact } from "../Artifacts.ts"; import { makeContainerRuntime } from "../runtime/Container.ts"; import { makeDatabaseSnapshots } from "../services/DatabaseSnapshot.ts"; import { makeDockerDatabaseStorage } from "./DockerDatabaseStorage.ts"; -const helperImage = - "public.ecr.aws/docker/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; +const postgresImage = (version: string) => + resolveArtifact({ service: "database", version: postgresVersion(version) }).pipe( + Effect.map(({ image }) => image), + ); const Marker = Schema.Struct({ backend: Schema.Literals(["docker", "host"]), volume: Schema.optionalKey(Schema.String), @@ -75,6 +78,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-" }); const stateRoot = path.join(root, "state"); const storageRoot = path.join(stateRoot, "stack", "data"); @@ -144,7 +148,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { helperImage, "/bin/sh", "-c", - `set -eu; printf 17 > ${quote(`/store/${sourceMarker.namespace}/data/PG_VERSION`)}; printf source > ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}`, + `set -eu; printf 17 > ${quote(`/store/${sourceMarker.namespace}/data/PG_VERSION`)}; printf source > ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}; /usr/bin/busybox setfattr -n user.storage-smoke -v preserved ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}`, ]); yield* source.markInitialized("17"); yield* fs.writeFileString( @@ -199,10 +203,10 @@ describe("Docker database storage", { timeout: 120_000 }, () => { helperImage, "/bin/sh", "-c", - `test "$(cat /store/fixture)" = source`, + `set -eu; test "$(cat /store/fixture)" = source; /usr/bin/busybox setfattr -x user.storage-smoke /store/fixture`, ]); - yield* target.removeData("17"); + yield* target.removeData("unsupported"); const resetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( yield* fs.readFileString(path.join(targetRoot, ".supabase-database-storage.json")), ); @@ -217,7 +221,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { helperImage, "/bin/sh", "-c", - 'descriptor=$(find /store -name descriptor.json -print -quit); printf corrupt > "$descriptor"', + 'descriptor=$(/usr/bin/busybox find /store -name descriptor.json -print -quit); printf corrupt > "$descriptor"', ]); const corrupt = yield* target.restoreSnapshot("17", "roundtrip").pipe( Effect.matchEffect({ @@ -243,6 +247,21 @@ describe("Docker database storage", { timeout: 120_000 }, () => { expect(missing).toSatisfy((exit) => Exit.isFailure(exit)); yield* target.removeData("17"); + yield* source.saveSnapshot("17", "retention-first"); + yield* source.saveSnapshot("17", "retention-second"); + yield* source.saveSnapshot("17", "retention-third"); + yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `test "$(/usr/bin/busybox find ${quote(`/store/${sourceMarker.cacheNamespace}/entries`)} -mindepth 1 -maxdepth 1 -type d | /usr/bin/busybox wc -l)" -eq 3`, + ]); + expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(false); + yield* source.saveSnapshot("17", "roundtrip"); yield* source.destroyData("17"); expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(true); yield* target.destroyData("17"); @@ -288,6 +307,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-legacy-" }); const storageRoot = path.join(root, "state", "stack", "data"); const cacheRoot = path.join(root, "cache"); @@ -402,6 +422,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-missing-" }); const storageRoot = path.join(root, "state", "stack", "data"); const cacheRoot = path.join(root, "cache"); @@ -480,6 +501,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-host-" }); const stateRoot = path.join(root, "state"); const storageRoot = path.join(stateRoot, "stack", "data"); @@ -578,7 +600,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { ]), ).toBe(expectedOwnership); yield* storage.removeData("17"); - yield* storage.destroyData("17"); + yield* storage.destroyData("unsupported"); expect(yield* fs.exists(path.join(instanceRoot, ".supabase-database-storage.json"))).toBe( true, ); @@ -594,6 +616,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-reopen-" }); const stateRoot = path.join(root, "state"); const storageRoot = path.join(stateRoot, "stack", "data"); diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index 6edde9b15a..a924bf1fc4 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -13,13 +13,11 @@ import { } from "effect"; import { ChildProcess } from "effect/unstable/process"; import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; +import { postgresVersion, resolveArtifact } from "../Artifacts.ts"; import type { ContainerRuntime } from "../runtime/Container.ts"; import type { DatabaseRuntime } from "../services/Database.ts"; import type { DockerHelperRegistry } from "./DockerHelperRegistry.ts"; -const HELPER_IMAGE = - "public.ecr.aws/docker/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; - const Marker = Schema.Struct({ backend: Schema.Literals(["docker", "host"]), volume: Schema.optionalKey(Schema.String), @@ -111,6 +109,9 @@ const errorFor = (operation: string, cause: unknown) => const shellQuote = (value: string): string => `'${value.replaceAll("'", "'\\''")}'`; +const withSnapshotLock = (root: string, command: string): string => + `set -eu; /usr/bin/busybox mkdir -p ${shellQuote(root)}; exec 9>${shellQuote(`${root}/.lock`)}; attempt=0; until lock_error=$(/usr/bin/busybox flock -n 9 2>&1); do if [ -n "$lock_error" ]; then echo "$lock_error" >&2; exit 1; fi; if [ "$attempt" -ge 120 ]; then echo 'Timed out waiting for database snapshot lock' >&2; exit 1; fi; attempt=$((attempt + 1)); /usr/bin/busybox sleep 1; done; /usr/bin/sh -eu -c ${shellQuote(command)}`; + const parseMajor = (version: string): number | undefined => { const major = Number(version.trim().split(".")[0]); return Number.isInteger(major) ? major : undefined; @@ -504,6 +505,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") }); const helperId = yield* Ref.make(undefined); + const helperImage = yield* Ref.make(undefined); const helperCleanupPending = yield* Ref.make(false); const operationLock = yield* Semaphore.make(1); const ownerScope = yield* Scope.Scope; @@ -536,6 +538,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ), ); yield* Ref.set(helperId, undefined); + yield* Ref.set(helperImage, undefined); yield* Ref.set(helperCleanupPending, false); } }), @@ -545,11 +548,24 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ownerScope, removeHelper().pipe(Effect.catch((cause) => Effect.logError(cause))), ); + const imageForVersion = (version: string, fallback: boolean) => + resolveArtifact({ service: "database", version: postgresVersion(version) }).pipe( + Effect.map(({ image }) => image), + Effect.catchTag("ArtifactError", (cause) => + fallback + ? resolveArtifact({ service: "database" }).pipe(Effect.map(({ image }) => image)) + : Effect.fail(cause), + ), + Effect.mapError((cause) => errorFor("helper", cause)), + ); const acquireHelper = Effect.fn("DockerDatabaseStorage.acquireHelper")( - (mounts: ReadonlyArray) => + (mounts: ReadonlyArray, image: string) => Effect.gen(function* () { const current = yield* Ref.get(helperId); - if (current !== undefined && (yield* Ref.get(helperCleanupPending))) + if ( + current !== undefined && + ((yield* Ref.get(helperCleanupPending)) || (yield* Ref.get(helperImage)) !== image) + ) yield* removeHelper(); const active = yield* Ref.get(helperId); if (active !== undefined) return active; @@ -563,7 +579,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* options.fs .makeDirectory(options.cacheRoot, { recursive: true }) .pipe(Effect.mapError((cause) => errorFor("helper", cause))); - yield* options.container.prepare(HELPER_IMAGE); + yield* options.container.prepare(image); const token = yield* options.crypto.randomUUIDv4.pipe( Effect.mapError((cause) => errorFor("helper", cause)), ); @@ -571,6 +587,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") // Register the deterministic owned name before the remote create starts so an // interrupted docker run can still be removed by the same scope. yield* Ref.set(helperId, name); + yield* Ref.set(helperImage, image); const created = yield* engineCommand([ "run", "-d", @@ -583,7 +600,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") "--label", `com.supabase.instance=${options.instanceId}`, ...mountArgs(mounts), - HELPER_IMAGE, + image, "/bin/sh", "-c", "trap : TERM INT; while :; do sleep 3600; done", @@ -595,17 +612,20 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ); const missingContainer = (message: string) => /no such (?:container|object)/iu.test(message); - const volumeHelperKey = (mounts: ReadonlyArray): string | undefined => { + const volumeHelperKey = ( + mounts: ReadonlyArray, + image: string, + ): string | undefined => { if (options.helpers === undefined) return undefined; if (mounts.length === 0 || mounts.some((mount) => (mount.type ?? "bind") !== "volume")) return undefined; - return mounts + return `${image}\0${mounts .map( (mount) => `${mount.source}\0${mount.target}\0${mount.volumeSubpath ?? ""}\0${mount.readOnly ? "ro" : "rw"}`, ) .sort() - .join("\n"); + .join("\n")}`; }; const closeSharedHelper = (id: string) => engineCommand(["rm", "-f", id]).pipe( @@ -617,7 +637,11 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") // Only the process that created the container removes it. `docker rm` without -f // fails while the container is running, so a racing creator cannot delete it. // A container left in `created` cannot exec; remove it and start another. - const openSharedHelper = (mounts: ReadonlyArray, key: string) => + const openSharedHelper = ( + mounts: ReadonlyArray, + key: string, + image: string, + ) => Effect.gen(function* () { const name = `supabase-db-helper-${(yield* hash(key)).slice(0, 32)}`; const status = yield* engineCommand([ @@ -648,7 +672,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") } if (options.container === undefined) return yield* errorFor("helper", "Container runtime is unavailable"); - yield* options.container.prepare(HELPER_IMAGE); + yield* options.container.prepare(image); return yield* engineCommand([ "run", "-d", @@ -661,7 +685,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") "--label", `com.supabase.stack=${options.stackId}`, ...mountArgs(mounts), - HELPER_IMAGE, + image, "/bin/sh", "-c", "trap : TERM INT; while :; do sleep 3600; done", @@ -677,44 +701,50 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") const runHelper = Effect.fn("DockerDatabaseStorage.helper")(( command: string, mounts: ReadonlyArray, + version: string, + fallback = false, ): Effect.Effect => { - const sharedKey = volumeHelperKey(mounts); - if (sharedKey !== undefined && options.helpers !== undefined) { - const helpers = options.helpers; - const exec = (id: string) => engineCommand(["exec", id, "/bin/sh", "-c", command]); - const open = openSharedHelper(mounts, sharedKey); - return helpers.use(sharedKey, open, closeSharedHelper, exec).pipe( - Effect.catchTag("DockerDatabaseStorageError", (cause) => - /no such container|is not running/iu.test(cause.message) - ? helpers - .drop(sharedKey) - .pipe(Effect.andThen(helpers.use(sharedKey, open, closeSharedHelper, exec))) - : Effect.fail(cause), - ), - Effect.mapError((cause) => errorFor("helper", cause)), - ); - } const cleanupAfterFailure = Effect.gen(function* () { if ((yield* Ref.get(helperId)) === undefined) return; yield* Ref.set(helperCleanupPending, true); yield* removeHelper(); }); - const operation = Effect.uninterruptibleMask((restore) => - restore( - Effect.gen(function* () { - const id = yield* acquireHelper(mounts); - return yield* engineCommand(["exec", id, "/bin/sh", "-c", command]); - }), - ).pipe( - Effect.onExit((exit) => - Exit.isFailure(exit) ? Effect.uninterruptible(cleanupAfterFailure) : Effect.void, - ), - Effect.mapError((cause) => errorFor("mount", cause)), - ), + return imageForVersion(version, fallback).pipe( + Effect.flatMap((image) => { + const sharedKey = volumeHelperKey(mounts, image); + if (sharedKey !== undefined && options.helpers !== undefined) { + const helpers = options.helpers; + const exec = (id: string) => engineCommand(["exec", id, "/bin/sh", "-c", command]); + const open = openSharedHelper(mounts, sharedKey, image); + return helpers.use(sharedKey, open, closeSharedHelper, exec).pipe( + Effect.catchTag("DockerDatabaseStorageError", (cause) => + /no such container|is not running/iu.test(cause.message) + ? helpers + .drop(sharedKey) + .pipe(Effect.andThen(helpers.use(sharedKey, open, closeSharedHelper, exec))) + : Effect.fail(cause), + ), + Effect.mapError((cause) => errorFor("helper", cause)), + ); + } + const operation = Effect.uninterruptibleMask((restore) => + restore( + Effect.gen(function* () { + const id = yield* acquireHelper(mounts, image); + return yield* engineCommand(["exec", id, "/bin/sh", "-c", command]); + }), + ).pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) ? Effect.uninterruptible(cleanupAfterFailure) : Effect.void, + ), + Effect.mapError((cause) => errorFor("mount", cause)), + ), + ); + return operationLock + .withPermit(operation) + .pipe(Effect.mapError((cause) => errorFor("helper", cause))); + }), ); - return operationLock - .withPermit(operation) - .pipe(Effect.mapError((cause) => errorFor("helper", cause))); }); const getMarker = Effect.gen(function* () { @@ -790,6 +820,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") { source: options.instanceRoot, target: "/instance", readOnly: false }, { source: options.cacheRoot, target: "/cache", readOnly: false }, ], + version, ); if (major.trim() !== majorVersion(version)) return yield* errorFor( @@ -807,6 +838,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") const major = yield* runHelper( `set -eu; if [ ! -f ${shellQuote(`${store}/data/PG_VERSION`)} ]; then echo 'Initialized PostgreSQL data is missing PG_VERSION' >&2; exit 1; fi; cat ${shellQuote(`${store}/data/PG_VERSION`)}`, [{ source: marker.volume ?? "", target: "/store", readOnly: false, type: "volume" }], + version, ); if (major.trim() !== majorVersion(version)) return yield* errorFor( @@ -817,6 +849,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* runHelper( `set -eu; mkdir -p ${shellQuote(`${store}/data`)} ${shellQuote(`${cache}/entries`)} ${shellQuote(`${cache}/stages`)}; chown -R 100:101 ${shellQuote(store)}`, [{ source: marker.volume ?? "", target: "/store", readOnly: false, type: "volume" }], + version, ); } }).pipe(Effect.mapError((cause) => errorFor("prepare", cause))), @@ -856,12 +889,13 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* runHelper( `set -eu; if [ ! -f ${shellQuote(versionFile)} ]; then echo 'Database readiness requires PG_VERSION' >&2; exit 1; fi; actual=$(cat ${shellQuote(versionFile)}); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Database PostgreSQL major does not match requested version' >&2; exit 1; fi`, snapshotPaths(marker).mounts, + version, ); } yield* writeMarker({ ...marker, initialized: true }); }).pipe(Effect.mapError((cause) => errorFor("ready", cause))), ); - const removeData = Effect.fn("DockerDatabaseStorage.removeData")((_version: string) => + const removeData = Effect.fn("DockerDatabaseStorage.removeData")((version: string) => Effect.gen(function* () { const markerOption = yield* getMarkerForRemoval; if (Option.isNone(markerOption)) return; @@ -870,11 +904,15 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* runHelper( `set -eu; rm -rf /instance/data /instance/.supabase-restore-*; mkdir -p /instance/data; chown 100:101 /instance/data`, snapshotPaths(marker).mounts, + version, + true, ); } else { yield* runHelper( `set -eu; rm -rf ${shellQuote(`/store/${marker.namespace}/data`)}; mkdir -p ${shellQuote(`/store/${marker.namespace}/data`)}; chown 100:101 ${shellQuote(`/store/${marker.namespace}/data`)}`, [{ source: marker.volume ?? "", target: "/store", readOnly: false, type: "volume" }], + version, + true, ); } yield* options.fs @@ -885,7 +923,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* writeMarker({ ...marker, initialized: false }); }).pipe(Effect.mapError((cause) => errorFor("reset", cause))), ); - const destroyData = Effect.fn("DockerDatabaseStorage.destroyData")((_version: string) => + const destroyData = Effect.fn("DockerDatabaseStorage.destroyData")((version: string) => Effect.gen(function* () { const present = yield* getMarkerIfPresent; const markerOption = Option.isSome(present) ? present : yield* getMarkerForRemoval; @@ -895,6 +933,8 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* runHelper( `set -eu; rm -rf /instance/data /instance/.supabase-restore-* /instance/.supabase-database-ready.json`, snapshotPaths(marker).mounts, + version, + true, ); yield* removeHelper(); } else { @@ -908,9 +948,12 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ), ); if (volumeExists) - yield* runHelper(`set -eu; rm -rf ${shellQuote(`/store/${marker.namespace}`)}`, [ - { source: marker.volume, target: "/store", readOnly: false, type: "volume" }, - ]); + yield* runHelper( + `set -eu; rm -rf ${shellQuote(`/store/${marker.namespace}`)}`, + [{ source: marker.volume, target: "/store", readOnly: false, type: "volume" }], + version, + true, + ); yield* removeHelper(); } }).pipe(Effect.mapError((cause) => errorFor("destroy", cause))), @@ -995,12 +1038,13 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") }); const cacheOwnership = store.backend === "host" - ? `; owner=$(stat -c "%u:%g" /cache); mkdir -p /cache/stack-database-snapshots-helper; chown "$owner" /cache/stack-database-snapshots-helper; chown -R "$owner" ${root}` + ? `; owner=$(/usr/bin/busybox stat -c "%u:%g" /cache); /usr/bin/busybox mkdir -p /cache/stack-database-snapshots-helper; /usr/bin/busybox chown "$owner" /cache/stack-database-snapshots-helper; /usr/bin/busybox chown -R "$owner" ${root}` : ""; - yield* runHelper( - `set -eu; mkdir -p ${root}; flock -x -w 120 ${shellQuote(`${root}/.lock`)} sh -eu -c ${shellQuote(`set -eu; mkdir -p ${root}/entries ${root}/stages; find ${root}/stages -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +; find ${root}/entries -mindepth 1 -maxdepth 1 -name '*.retired' -exec rm -rf -- {} +; trap 'rm -rf ${stage}${cacheOwnership}' EXIT; if [ -e ${source}/postmaster.pid ]; then echo 'Cannot save a running database snapshot' >&2; exit 1; fi; if [ ! -f ${source}/PG_VERSION ]; then echo 'Cannot save snapshot: PG_VERSION is missing' >&2; exit 1; fi; actual=$(cat ${source}/PG_VERSION); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Cannot save snapshot: PostgreSQL major does not match requested version' >&2; exit 1; fi; bad=$(find ${source} \\( ! -type f ! -type d \\) -print -quit); if [ -n "$bad" ]; then echo "Cannot save snapshot: unsupported filesystem entry $bad" >&2; exit 1; fi; rm -rf ${stage}; mkdir -p ${stage}; cp -a --reflink=auto ${source} ${stage}/data; printf '%s' ${shellQuote(descriptor)} > ${stage}/descriptor.json; if [ -e ${target} ]; then rm -rf ${target}.retired; mv ${target} ${target}.retired; if ! mv ${stage} ${target}; then mv ${target}.retired ${target}; exit 1; fi; else mv ${stage} ${target}; fi; rm -rf ${target}.retired; touch ${target}; find ${root}/entries -mindepth 1 -maxdepth 1 -type d ! -name ${digest} ! -name '*.retired' -printf '%T@ %p\\n' | sort -rn | tail -n +3 | cut -d' ' -f2- | xargs -r rm -rf`)}; rm -rf ${shellQuote(stage)}`, - paths.mounts, + const command = withSnapshotLock( + root, + `set -eu; /usr/bin/busybox mkdir -p ${shellQuote(`${root}/entries`)} ${shellQuote(`${root}/stages`)}; /usr/bin/busybox find ${shellQuote(`${root}/stages`)} -mindepth 1 -maxdepth 1 -exec /usr/bin/busybox rm -rf -- {} +; /usr/bin/busybox find ${shellQuote(`${root}/entries`)} -mindepth 1 -maxdepth 1 -name '*.retired' -exec /usr/bin/busybox rm -rf -- {} +; trap '/usr/bin/busybox rm -rf ${stage}${cacheOwnership}' EXIT; if [ -e ${shellQuote(`${source}/postmaster.pid`)} ]; then echo 'Cannot save a running database snapshot' >&2; exit 1; fi; if [ ! -f ${shellQuote(`${source}/PG_VERSION`)} ]; then echo 'Cannot save snapshot: PG_VERSION is missing' >&2; exit 1; fi; actual=$(/usr/bin/busybox cat ${shellQuote(`${source}/PG_VERSION`)}); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Cannot save snapshot: PostgreSQL major does not match requested version' >&2; exit 1; fi; bad=$(/usr/bin/busybox find ${shellQuote(source)} \\( ! -type f ! -type d \\) -print -quit); if [ -n "$bad" ]; then echo "Cannot save snapshot: unsupported filesystem entry $bad" >&2; exit 1; fi; /usr/bin/busybox rm -rf ${shellQuote(stage)}; /usr/bin/busybox mkdir -p ${shellQuote(stage)}; /usr/local/bin/cp -a --reflink=auto ${shellQuote(source)} ${shellQuote(`${stage}/data`)}; printf '%s' ${shellQuote(descriptor)} > ${shellQuote(`${stage}/descriptor.json`)}; if [ -e ${shellQuote(target)} ]; then /usr/bin/busybox rm -rf ${shellQuote(`${target}.retired`)}; /usr/bin/busybox mv ${shellQuote(target)} ${shellQuote(`${target}.retired`)}; if ! /usr/bin/busybox mv ${shellQuote(stage)} ${shellQuote(target)}; then /usr/bin/busybox mv ${shellQuote(`${target}.retired`)} ${shellQuote(target)}; exit 1; fi; else /usr/bin/busybox mv ${shellQuote(stage)} ${shellQuote(target)}; fi; /usr/bin/busybox rm -rf ${shellQuote(`${target}.retired`)}; /usr/bin/busybox touch ${shellQuote(target)}; /usr/bin/busybox find ${shellQuote(`${root}/entries`)} -mindepth 1 -maxdepth 1 -type d ! -name ${shellQuote(digest)} ! -name '*.retired' -exec /usr/bin/busybox stat -c '%y %n' {} + | /usr/bin/busybox sort -r | /usr/bin/busybox tail -n +3 | /usr/bin/busybox cut -d ' ' -f 4- | /usr/bin/busybox xargs -r /usr/bin/busybox rm -rf`, ); + yield* runHelper(command, paths.mounts, version); }).pipe(Effect.mapError((cause) => errorFor("snapshot", cause))), ); const restoreSnapshot = Effect.fn("DockerDatabaseStorage.restoreSnapshot")( @@ -1029,15 +1073,18 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") profile: "supabase", keyDigest: digest, }); - const targetSetup = store.initialized ? `test -d ${data}` : `mkdir -p ${data}`; + const targetSetup = store.initialized + ? `test -d ${shellQuote(data)}` + : `/usr/bin/busybox mkdir -p ${shellQuote(data)}`; const cacheOwnership = store.backend === "host" - ? `; owner=$(stat -c "%u:%g" /cache); mkdir -p /cache/stack-database-snapshots-helper; chown "$owner" /cache/stack-database-snapshots-helper; chown -R "$owner" ${root}` + ? `; owner=$(/usr/bin/busybox stat -c "%u:%g" /cache); /usr/bin/busybox mkdir -p /cache/stack-database-snapshots-helper; /usr/bin/busybox chown "$owner" /cache/stack-database-snapshots-helper; /usr/bin/busybox chown -R "$owner" ${root}` : ""; - const result = yield* runHelper( - `set -eu; mkdir -p ${root}; flock -x -w 120 ${shellQuote(`${root}/.lock`)} sh -eu -c ${shellQuote(`set -eu; mkdir -p ${root}/entries ${root}/stages; find ${root}/stages -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +; find ${root}/entries -mindepth 1 -maxdepth 1 -name '*.retired' -exec rm -rf -- {} +; trap 'rm -rf ${stage}${cacheOwnership}' EXIT; if ! ${targetSetup}; then echo 'Initialized restore target data directory is missing' >&2; exit 1; fi; bad=$(find ${data} -mindepth 1 -print -quit); if [ -n "$bad" ]; then echo NONEMPTY; exit 0; fi; if [ ! -d ${source} ]; then echo MISS; exit 0; fi; if [ ! -f ${source}/descriptor.json ]; then echo 'Snapshot descriptor is missing' >&2; exit 1; fi; actual=$(cat ${source}/descriptor.json); expected=${shellQuote(descriptor)}; if [ "$actual" != "$expected" ]; then echo 'Snapshot descriptor does not match requested identity' >&2; exit 1; fi; bad=$(find ${source}/data \\( ! -type f ! -type d \\) -print -quit); if [ -n "$bad" ]; then echo "Snapshot contains unsupported filesystem entry $bad" >&2; exit 1; fi; if [ -e ${source}/data/postmaster.pid ]; then echo 'Snapshot contains postmaster.pid' >&2; exit 1; fi; rm -rf ${stage}; cp -a --reflink=auto ${source}/data ${stage}; actual=$(cat ${stage}/PG_VERSION); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Snapshot PostgreSQL major does not match requested version' >&2; exit 1; fi; ${store.backend === "host" ? `chown -R 100:101 ${stage};` : ""} rmdir ${data}; mv ${stage} ${data}; touch ${source}; echo HIT`)}; rm -rf ${shellQuote(stage)}`, - paths.mounts, + const command = withSnapshotLock( + root, + `set -eu; /usr/bin/busybox mkdir -p ${shellQuote(`${root}/entries`)} ${shellQuote(`${root}/stages`)}; /usr/bin/busybox find ${shellQuote(`${root}/stages`)} -mindepth 1 -maxdepth 1 -exec /usr/bin/busybox rm -rf -- {} +; /usr/bin/busybox find ${shellQuote(`${root}/entries`)} -mindepth 1 -maxdepth 1 -name '*.retired' -exec /usr/bin/busybox rm -rf -- {} +; trap '/usr/bin/busybox rm -rf ${stage}${cacheOwnership}' EXIT; if ! ${targetSetup}; then echo 'Initialized restore target data directory is missing' >&2; exit 1; fi; bad=$(/usr/bin/busybox find ${shellQuote(data)} -mindepth 1 -print -quit); if [ -n "$bad" ]; then echo NONEMPTY; exit 0; fi; if [ ! -d ${shellQuote(source)} ]; then echo MISS; exit 0; fi; if [ ! -f ${shellQuote(`${source}/descriptor.json`)} ]; then echo 'Snapshot descriptor is missing' >&2; exit 1; fi; actual=$(/usr/bin/busybox cat ${shellQuote(`${source}/descriptor.json`)}); expected=${shellQuote(descriptor)}; if [ "$actual" != "$expected" ]; then echo 'Snapshot descriptor does not match requested identity' >&2; exit 1; fi; bad=$(/usr/bin/busybox find ${shellQuote(`${source}/data`)} \\( ! -type f ! -type d \\) -print -quit); if [ -n "$bad" ]; then echo "Snapshot contains unsupported filesystem entry $bad" >&2; exit 1; fi; if [ -e ${shellQuote(`${source}/data/postmaster.pid`)} ]; then echo 'Snapshot contains postmaster.pid' >&2; exit 1; fi; /usr/bin/busybox rm -rf ${shellQuote(stage)}; /usr/local/bin/cp -a --reflink=auto ${shellQuote(`${source}/data`)} ${shellQuote(stage)}; actual=$(/usr/bin/busybox cat ${shellQuote(`${stage}/PG_VERSION`)}); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Snapshot PostgreSQL major does not match requested version' >&2; exit 1; fi; ${store.backend === "host" ? `/usr/bin/busybox chown -R 100:101 ${shellQuote(stage)};` : ""} /usr/bin/busybox rmdir ${shellQuote(data)}; /usr/bin/busybox mv ${shellQuote(stage)} ${shellQuote(data)}; /usr/bin/busybox touch ${shellQuote(source)}; echo HIT`, ); + const result = yield* runHelper(command, paths.mounts, version); if (result === "MISS") return false; if (result === "NONEMPTY") return yield* errorFor("restore", "Restore target data directory must be empty"); From 45c6f277481d8e98b5ad8a0b5aa72a9cef508bff Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 23 Sep 2026 19:08:35 +0200 Subject: [PATCH 14/20] test(stack): cover postgres 15 storage snapshots --- .../DockerDatabaseStorage.integration.test.ts | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts index 5566992e29..739063d3eb 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts @@ -72,6 +72,108 @@ const docker = Effect.fn("DockerStorageTest.docker")((args: ReadonlyArray `'${value.replaceAll("'", "'\\''")}'`; describe("Docker database storage", { timeout: 120_000 }, () => { + it.live("round-trips PostgreSQL 15 data with its catalog image", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("15"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-pg15-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const sourceRoot = path.join(storageRoot, "source"); + const targetRoot = path.join(storageRoot, "target"); + yield* fs.makeDirectory(sourceRoot, { recursive: true }); + yield* fs.makeDirectory(targetRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ engine: "docker" }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-pg15-${yield* crypto.randomUUIDv4}`; + const makeStorage = (instanceId: string, instanceRoot: string) => + makeDockerDatabaseStorage({ + runtime: "docker", + stackId, + instanceId, + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + const source = yield* makeStorage("source", sourceRoot); + const target = yield* makeStorage("target", targetRoot); + const ownerScope = yield* Scope.Scope; + yield* Scope.addFinalizer( + ownerScope, + Effect.gen(function* () { + yield* source.destroyData("15").pipe(Effect.ignore); + yield* target.destroyData("15").pipe(Effect.ignore); + const marker = yield* fs + .readFileString(path.join(sourceRoot, ".supabase-database-storage.json")) + .pipe(Effect.option); + if (Option.isSome(marker)) { + const parsed = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + marker.value, + ).pipe(Effect.option); + if (Option.isSome(parsed) && parsed.value.volume !== undefined) + yield* docker(["volume", "rm", parsed.value.volume]).pipe(Effect.ignore); + } + }).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.ignore, + ), + ); + + yield* source.prepare("15"); + const sourceMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), + ); + if (sourceMarker.backend !== "docker" || sourceMarker.volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `set -eu; printf 15 > ${quote(`/store/${sourceMarker.namespace}/data/PG_VERSION`)}; printf pg15-source > ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}`, + ]); + yield* source.markInitialized("15"); + yield* fs.writeFileString( + path.join(sourceRoot, ".supabase-database-ready.json"), + '{"version":"15","runtime":"docker","profile":"supabase"}', + ); + yield* source.saveSnapshot("15", "restore-me"); + + yield* target.prepare("15"); + expect(yield* target.restoreSnapshot("15", "restore-me")).toBe(true); + const targetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(targetRoot, ".supabase-database-storage.json")), + ); + expect(targetMarker.volume).toBe(sourceMarker.volume); + yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store,volume-subpath=${targetMarker.namespace}/data`, + helperImage, + "/bin/sh", + "-c", + 'test "$(cat /store/PG_VERSION)" = 15 && test "$(cat /store/fixture)" = pg15-source', + ]); + yield* target.destroyData("15"); + yield* source.destroyData("15"); + yield* docker(["volume", "rm", sourceMarker.volume]); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + it.live("round-trips stopped data and protects managed namespaces", () => Effect.scoped( Effect.gen(function* () { From e9b11ad8a8f3eb8c036188ab45d347563e8de00d Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 23 Sep 2026 20:58:19 +0200 Subject: [PATCH 15/20] test(stack): satisfy Effect lint in the partial-copy fixture The node:fs imports fail the required quality check. --- .../storage/DirectoryCopy.integration.test.ts | 45 ++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/packages/stack/src/storage/DirectoryCopy.integration.test.ts b/packages/stack/src/storage/DirectoryCopy.integration.test.ts index faa612af47..f0be2e9cb5 100644 --- a/packages/stack/src/storage/DirectoryCopy.integration.test.ts +++ b/packages/stack/src/storage/DirectoryCopy.integration.test.ts @@ -1,5 +1,3 @@ -import { chmodSync, mkdirSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { @@ -173,7 +171,7 @@ describe("copyDirectory", () => { yield* copyDirectory(source, destination).pipe( Effect.provideService( ChildProcessSpawner.ChildProcessSpawner, - failedHostCopy(destination), + failedHostCopy(fs, path, destination), ), ); @@ -198,7 +196,7 @@ describe("copyDirectory", () => { const fiber = yield* copyDirectory(source, destination).pipe( Effect.provideService( ChildProcessSpawner.ChildProcessSpawner, - hangingCopy(started, destination), + hangingCopy(fs, path, started, destination), ), Effect.forkChild, ); @@ -238,7 +236,7 @@ describe("copyDirectory", () => { Effect.provideService(FileSystem.FileSystem, pausing), Effect.provideService( ChildProcessSpawner.ChildProcessSpawner, - failedHostCopy(destination), + failedHostCopy(fs, path, destination), ), Effect.forkChild, ); @@ -272,12 +270,13 @@ const processHandle = (exitCode: Effect.Effect) => unref: Effect.succeed(Effect.void), }); -const writePartial = (destination: string) => { - const nested = join(destination, "nested"); - mkdirSync(nested, { recursive: true }); - writeFileSync(join(nested, "child.txt"), "partial\n"); - if (process.platform !== "win32") chmodSync(nested, 0o555); -}; +const writePartial = (fs: FileSystem.FileSystem, path: Path.Path, destination: string) => + Effect.gen(function* () { + const nested = path.join(destination, "nested"); + yield* fs.makeDirectory(nested, { recursive: true }); + yield* fs.writeFileString(path.join(nested, "child.txt"), "partial\n"); + if (process.platform !== "win32") yield* fs.chmod(nested, 0o555); + }); const hostCommand = (command: ChildProcess.Command) => { if (!ChildProcess.isStandardCommand(command)) return Effect.die("unexpected piped command"); @@ -287,23 +286,27 @@ const hostCommand = (command: ChildProcess.Command) => { return undefined; }; -const hangingCopy = (started: Deferred.Deferred, destination: string) => +const hangingCopy = ( + fs: FileSystem.FileSystem, + path: Path.Path, + started: Deferred.Deferred, + destination: string, +) => ChildProcessSpawner.make((command) => { const find = hostCommand(command); if (find !== undefined) return find; - return Effect.sync(() => { - writePartial(destination); - return processHandle(Effect.never); - }).pipe(Effect.tap(() => Deferred.succeed(started, undefined))); + return writePartial(fs, path, destination).pipe( + Effect.as(processHandle(Effect.never)), + Effect.tap(() => Deferred.succeed(started, undefined)), + ); }); // Exit 8 is a real robocopy failure and a non-zero cp status, not a usage error. -const failedHostCopy = (destination: string) => +const failedHostCopy = (fs: FileSystem.FileSystem, path: Path.Path, destination: string) => ChildProcessSpawner.make((command) => { const find = hostCommand(command); if (find !== undefined) return find; - return Effect.sync(() => { - writePartial(destination); - return processHandle(Effect.succeed(ChildProcessSpawner.ExitCode(8))); - }); + return writePartial(fs, path, destination).pipe( + Effect.as(processHandle(Effect.succeed(ChildProcessSpawner.ExitCode(8)))), + ); }); From 31ba99cdf65ff92679726a283cc4cbf1c7cacd27 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Thu, 24 Sep 2026 09:55:04 +0200 Subject: [PATCH 16/20] fix(stack): reject Windows junctions during snapshot copy --- .github/workflows/test.yml | 1 + .../storage/DirectoryCopy.integration.test.ts | 35 +++++++++++++++++++ packages/stack/src/storage/DirectoryCopy.ts | 10 +++++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9b60e8622d..bb91680cb0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -149,6 +149,7 @@ jobs: run: >- pnpm test:integration src/Ports.integration.test.ts + src/storage/DirectoryCopy.integration.test.ts src/State.integration.test.ts src/State.process.integration.test.ts src/HostProcess.integration.test.ts diff --git a/packages/stack/src/storage/DirectoryCopy.integration.test.ts b/packages/stack/src/storage/DirectoryCopy.integration.test.ts index f0be2e9cb5..31e9dbef3c 100644 --- a/packages/stack/src/storage/DirectoryCopy.integration.test.ts +++ b/packages/stack/src/storage/DirectoryCopy.integration.test.ts @@ -1,7 +1,10 @@ import { NodeServices } from "@effect/platform-node"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- Windows junctions need Node's junction option. +import { symlink as nativeSymlink } from "node:fs/promises"; import { describe, expect, it } from "@effect/vitest"; import { Cause, + Data, Deferred, Effect, Exit, @@ -23,6 +26,11 @@ const run = ( >, ) => Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); +class JunctionTestError extends Data.TaggedError("JunctionTestError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + describe("copyDirectory", () => { it.live("copies nested files and preserves modes", () => run( @@ -95,6 +103,33 @@ describe("copyDirectory", () => { ), ); + it.live("rejects Windows directory junctions without copying their targets", () => + run( + Effect.gen(function* () { + if (process.platform !== "win32") return; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "directory-copy-junction-" }); + const source = path.join(root, "source"); + const outside = path.join(root, "outside"); + const destination = path.join(root, "destination"); + yield* fs.makeDirectory(source); + yield* fs.makeDirectory(outside); + yield* fs.writeFileString(path.join(outside, "secret.txt"), "outside\n"); + yield* Effect.tryPromise({ + try: () => nativeSymlink(outside, path.join(source, "junction"), "junction"), + catch: (cause) => new JunctionTestError({ message: "create directory junction", cause }), + }); + + const result = yield* copyDirectory(source, destination).pipe(Effect.exit); + + expect(Exit.isFailure(result)).toBe(true); + expect(yield* fs.exists(destination)).toBe(false); + expect(yield* fs.readFileString(path.join(outside, "secret.txt"))).toBe("outside\n"); + }), + ), + ); + it.live("rejects a file source and leaves the destination absent", () => run( Effect.gen(function* () { diff --git a/packages/stack/src/storage/DirectoryCopy.ts b/packages/stack/src/storage/DirectoryCopy.ts index a5c2b4dd53..8b84843c4b 100644 --- a/packages/stack/src/storage/DirectoryCopy.ts +++ b/packages/stack/src/storage/DirectoryCopy.ts @@ -311,7 +311,14 @@ const scanUnsupported = ( }); for (const entry of entries) { if (entry.isSymbolicLink() || (!entry.isDirectory() && !entry.isFile())) return true; - if (entry.isDirectory()) pending.push(path.join(directory, entry.name)); + if (entry.isDirectory()) { + const entryPath = path.join(directory, entry.name); + const stats = yield* nativeLstat(entryPath).pipe( + Effect.mapError((cause) => errorFor("validate", source, destination, cause)), + ); + if (stats.isSymbolicLink()) return true; + pending.push(entryPath); + } } current = pending.pop(); } @@ -416,6 +423,7 @@ const hostCopies = (source: string, destination: string): ReadonlyArray Date: Thu, 24 Sep 2026 10:36:41 +0200 Subject: [PATCH 17/20] fix(stack): isolate helpers and await shadow teardown --- apps/cli/src/command-internal/stack-shadow.ts | 2 +- packages/stack/src/Owner.ts | 5 +- .../src/runtime/Container.integration.test.ts | 42 +++++++ packages/stack/src/runtime/Container.ts | 38 ++----- packages/stack/src/services/Database.ts | 6 +- .../DockerDatabaseStorage.integration.test.ts | 104 +++++++++++++++++- .../src/storage/DockerDatabaseStorage.ts | 12 +- .../stack/src/storage/DockerHelperRegistry.ts | 8 +- .../storage/DockerHelperRegistry.unit.test.ts | 6 +- 9 files changed, 173 insertions(+), 50 deletions(-) diff --git a/apps/cli/src/command-internal/stack-shadow.ts b/apps/cli/src/command-internal/stack-shadow.ts index 373e44000d..0b307a56ce 100644 --- a/apps/cli/src/command-internal/stack-shadow.ts +++ b/apps/cli/src/command-internal/stack-shadow.ts @@ -89,7 +89,7 @@ const initialize = Effect.fn("StackShadow.initialize")(function* ( jwtSecret: Redacted.make(input.jwtSecret), jwtExpiry: input.jwtExpiry, healthTimeoutMs: input.healthTimeoutSeconds * 1_000, - // Destroy issues stop without waiting. The stop before a snapshot still checkpoints. + // The disposable shadow can skip durability work; an explicit stop still checkpoints. stopGraceSeconds: 0, ...(input.rootKey === undefined ? {} : { rootKey: Redacted.make(input.rootKey) }), settings: dbSettings, diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index fabad493f8..c5650709a5 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -158,7 +158,10 @@ const makeOwnerWithDependencies = ( const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const http = yield* HttpClient.HttpClient; const ownerScope = yield* Scope.Scope; - const helpers = yield* makeDockerHelperRegistry; + const helperOwnerId = yield* crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => errorFor("identity", cause)), + ); + const helpers = yield* makeDockerHelperRegistry(helperOwnerId); const recipes = yield* Ref.make(new Map()); const instances = yield* Ref.make(new Map>()); const namespaces = yield* Ref.make(new Map()); diff --git a/packages/stack/src/runtime/Container.integration.test.ts b/packages/stack/src/runtime/Container.integration.test.ts index a2e8bdbaf0..945f04526a 100644 --- a/packages/stack/src/runtime/Container.integration.test.ts +++ b/packages/stack/src/runtime/Container.integration.test.ts @@ -123,6 +123,32 @@ describe("container process adapter", () => { ).pipe(Effect.provide(NodeServices.layer)), ); + it.live("waits until a discarded container stops before returning", () => + Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ engine: "docker" }); + yield* runtime.prepare(image); + const process = yield* runtime.launch({ + image, + stackId: "f".repeat(64), + instanceId: "discard-waits-for-stop", + env: {}, + args: [ + "-e", + "process.on('SIGTERM', () => setTimeout(() => process.exit(0), 2000)); setInterval(() => {}, 1000); console.log('ready')", + ], + }); + const logs = yield* ready(process); + + yield* process.discard; + expect(yield* running(process.id)).toBe(false); + expect(yield* exists(process.id)).toBe(true); + + yield* process.remove; + expect(yield* exists(process.id)).toBe(false); + yield* Fiber.interrupt(logs); + }).pipe(Effect.provide(NodeServices.layer)), + ); + it.live("returns cleanup authority when container start fails", () => Effect.scoped( Effect.gen(function* () { @@ -206,6 +232,22 @@ const exists = (id: string) => return Number(yield* child.exitCode) === 0; }); +const running = (id: string) => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + ChildProcess.make("docker", ["inspect", "--format={{.State.Running}}", id], { + stdin: "ignore", + }), + ); + const [output, exitCode] = yield* Effect.all( + [child.stdout.pipe(Stream.decodeText, Stream.mkString), child.exitCode], + { concurrency: "unbounded" }, + ); + expect(Number(exitCode)).toBe(0); + return output.trim() === "true"; + }); + const get = (port: number | undefined, path = "/") => { if (port === undefined) return Effect.fail(new ContainerTestError({ message: "container port was not assigned" })); diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index 1f755cfb34..2f1287ae8c 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -53,8 +53,7 @@ export interface ContainerProcess { readonly exitCode: Effect.Effect; readonly stdin: Sink.Sink; readonly stop: Effect.Effect; - /** Starts `stop` and a later `rm` without waiting for either to finish. */ - readonly beginStop: Effect.Effect; + readonly discard: Effect.Effect; readonly kill: Effect.Effect; readonly remove: Effect.Effect; } @@ -266,34 +265,11 @@ export const makeContainerRuntime = (options: { yield* run(["stop", "--time", grace, id]); yield* Ref.set(stopped, true); }); - // Issues stop, then removes the container, without waiting. The caller continues. - const beginStop = Effect.scoped( - Effect.gen(function* () { - if ((yield* Ref.get(removed)) || (yield* Ref.get(stopped))) return; - const script = `${options.engine} stop --time 10 ${id} && ${options.engine} rm ${id}`; - const child = yield* Effect.mapError( - spawner.spawn( - ChildProcess.make( - process.platform === "win32" ? "cmd" : "sh", - process.platform === "win32" ? ["/d", "/c", script] : ["-c", script], - { - detached: true, - stdin: "ignore", - stdout: "ignore", - stderr: "ignore", - }, - ), - ), - (cause) => errorFor("stop", cause), - ); - // `unref` succeeds with a reref effect. Drop it so the stop outlives this scope. - yield* Effect.asVoid( - Effect.mapError(child.unref, (cause) => errorFor("stop", cause)), - ); - yield* Ref.set(stopped, true); - yield* Ref.set(removed, true); - }), - ); + const discard = Effect.gen(function* () { + if ((yield* Ref.get(removed)) || (yield* Ref.get(stopped))) return; + yield* run(["stop", "--time", "0", id]); + yield* Ref.set(stopped, true); + }); const kill = Effect.gen(function* () { if ((yield* Ref.get(removed)) || (yield* Ref.get(stopped))) return; yield* run(["kill", id]); @@ -319,7 +295,7 @@ export const makeContainerRuntime = (options: { exitCode: Effect.fail(errorFor("wait", "Container did not start")), stdin: Sink.fail(errorFor("stdin", "Container did not start")), stop, - beginStop, + discard, kill, remove, }; diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index 161f742a67..6a088b2352 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -63,7 +63,7 @@ export const DatabaseConfig = Schema.Struct({ jwtSecret: Schema.Redacted(Schema.String), jwtExpiry: Schema.Finite, healthTimeoutMs: Schema.optionalKey(Schema.Finite), - /** When 0, destroy issues stop in the background. An explicit stop still waits. */ + /** When 0, the database is disposable and can use reduced-durability settings. */ stopGraceSeconds: Schema.optionalKey(Schema.Finite), rootKey: Schema.optionalKey(Schema.Redacted(Schema.String)), settings: Schema.optionalKey( @@ -143,7 +143,7 @@ const postgresArguments = (config: DatabaseConfig): Array => { "-c", `${key}=${String(value)}`, ]); - // stopGraceSeconds 0 is the discarded shadow. Crash safety only adds wait. + // Shadow databases are discarded, so they can use reduced-durability settings. if (config.stopGraceSeconds === 0) for (const setting of ["fsync=off", "synchronous_commit=off", "full_page_writes=off"]) if (!configured.has(setting.slice(0, setting.indexOf("=")))) settings.push("-c", setting); @@ -358,7 +358,7 @@ const runtimeFromContainer = (process: ContainerProcess, discard: boolean): Runt stop: process.stop.pipe(Effect.mapError((cause) => errorFor("stop", cause))), ...(discard ? { - discard: process.beginStop.pipe(Effect.mapError((cause) => errorFor("stop", cause))), + discard: process.discard.pipe(Effect.mapError((cause) => errorFor("stop", cause))), } : {}), remove: process.remove.pipe(Effect.mapError((cause) => errorFor("remove", cause))), diff --git a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts index 697f150baa..17dbd21003 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts @@ -16,6 +16,8 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { makeContainerRuntime } from "../runtime/Container.ts"; import { makeDatabaseSnapshots } from "../services/DatabaseSnapshot.ts"; import { makeDockerDatabaseStorage } from "./DockerDatabaseStorage.ts"; +import { makeDockerHelperRegistry } from "./DockerHelperRegistry.ts"; +import type { DockerHelperRegistry } from "./DockerHelperRegistry.ts"; const helperImage = "public.ecr.aws/docker/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; @@ -354,6 +356,11 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const stackId = `storage-helper-recovery-${yield* crypto.randomUUIDv4}`; const instanceId = "recovery"; + const helperScope = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(helperScope, Exit.void)); + const helperRegistry = yield* makeDockerHelperRegistry(yield* crypto.randomUUIDv4).pipe( + Effect.provideService(Scope.Scope, helperScope), + ); const storage = yield* makeDockerDatabaseStorage({ runtime: "docker", stackId, @@ -366,6 +373,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { crypto, container, spawner, + helpers: helperRegistry, }); yield* storage.prepare("17"); const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( @@ -378,8 +386,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { "ps", "--filter", "label=com.supabase.stack=" + stackId, - "--filter", - "label=com.supabase.instance=" + instanceId, "--format", "{{.Names}}", ]); @@ -387,15 +393,105 @@ describe("Docker database storage", { timeout: 120_000 }, () => { if (helper === undefined) return yield* new DockerTestError({ message: "Owned helper was not discoverable" }); yield* docker(["rm", "-f", helper]); - const failed = yield* storage.prepare("17").pipe(Effect.exit); - expect(failed).toSatisfy((exit) => Exit.isFailure(exit)); yield* storage.prepare("17"); yield* storage.destroyData("17"); + yield* Scope.close(helperScope, Exit.void); yield* docker(["volume", "rm", volume]); }), ).pipe(Effect.provide(NodeServices.layer)), ); + it.live("keeps another owner's volume helper running when one owner closes", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-helper-owner-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "database"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ engine: "docker" }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-helper-owner-${yield* crypto.randomUUIDv4}`; + const firstScope = yield* Scope.make(); + const secondScope = yield* Scope.make(); + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + yield* Scope.close(secondScope, Exit.void); + yield* Scope.close(firstScope, Exit.void); + const markerText = yield* fs + .readFileString(path.join(instanceRoot, ".supabase-database-storage.json")) + .pipe(Effect.option); + if (Option.isNone(markerText)) return; + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + markerText.value, + ).pipe(Effect.option); + if (Option.isSome(marker) && marker.value.volume !== undefined) + yield* docker(["volume", "rm", marker.value.volume]).pipe(Effect.ignore); + }).pipe(Effect.ignore), + ); + const firstHelpers = yield* makeDockerHelperRegistry("first-owner").pipe( + Effect.provideService(Scope.Scope, firstScope), + ); + const secondHelpers = yield* makeDockerHelperRegistry("second-owner").pipe( + Effect.provideService(Scope.Scope, secondScope), + ); + const makeStorage = (helpers: DockerHelperRegistry) => + makeDockerDatabaseStorage({ + runtime: "docker", + stackId, + instanceId: "database", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + helpers, + }); + const firstStorage = yield* makeStorage(firstHelpers).pipe( + Effect.provideService(Scope.Scope, firstScope), + ); + const secondStorage = yield* makeStorage(secondHelpers).pipe( + Effect.provideService(Scope.Scope, secondScope), + ); + yield* firstStorage.prepare("17"); + yield* secondStorage.prepare("17"); + const runningHelpers = (yield* docker([ + "ps", + "--filter", + `label=com.supabase.stack=${stackId}`, + "--format", + "{{.Names}}", + ])) + .split("\n") + .filter((name) => name.length > 0); + expect(runningHelpers).toHaveLength(2); + + yield* Scope.close(firstScope, Exit.void); + const remainingHelpers = (yield* docker([ + "ps", + "--filter", + `label=com.supabase.stack=${stackId}`, + "--format", + "{{.Names}}", + ])) + .split("\n") + .filter((name) => name.length > 0); + expect(remainingHelpers).toHaveLength(1); + expect(runningHelpers).toContain(remainingHelpers[0]); + yield* secondStorage.prepare("17"); + yield* secondStorage.destroyData("17"); + yield* Scope.close(secondScope, Exit.void); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + it.live("destroys an initialized storage after its volume is removed", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index 6edde9b15a..b935602497 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -614,12 +614,14 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ), Effect.asVoid, ); - // Only the process that created the container removes it. `docker rm` without -f - // fails while the container is running, so a racing creator cannot delete it. // A container left in `created` cannot exec; remove it and start another. - const openSharedHelper = (mounts: ReadonlyArray, key: string) => + const openSharedHelper = ( + mounts: ReadonlyArray, + key: string, + ownerId: string, + ) => Effect.gen(function* () { - const name = `supabase-db-helper-${(yield* hash(key)).slice(0, 32)}`; + const name = `supabase-db-helper-${(yield* hash(`${ownerId}\0${key}`)).slice(0, 32)}`; const status = yield* engineCommand([ "inspect", "--format", @@ -682,7 +684,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") if (sharedKey !== undefined && options.helpers !== undefined) { const helpers = options.helpers; const exec = (id: string) => engineCommand(["exec", id, "/bin/sh", "-c", command]); - const open = openSharedHelper(mounts, sharedKey); + const open = openSharedHelper(mounts, sharedKey, helpers.ownerId); return helpers.use(sharedKey, open, closeSharedHelper, exec).pipe( Effect.catchTag("DockerDatabaseStorageError", (cause) => /no such container|is not running/iu.test(cause.message) diff --git a/packages/stack/src/storage/DockerHelperRegistry.ts b/packages/stack/src/storage/DockerHelperRegistry.ts index c3bf659cbb..10baf5d071 100644 --- a/packages/stack/src/storage/DockerHelperRegistry.ts +++ b/packages/stack/src/storage/DockerHelperRegistry.ts @@ -7,6 +7,8 @@ interface DockerHelperHandle { } export interface DockerHelperRegistry { + /** Identifies helpers owned by one host process. */ + readonly ownerId: string; /** Runs `body` with the helper for `key`, opening it once per host lifetime. */ readonly use: ( key: string, @@ -22,7 +24,9 @@ export interface DockerHelperRegistry { * One sleeping container per volume mount. Mounts are fixed when the container is created, * and every database in a state directory uses that same volume. */ -export const makeDockerHelperRegistry: Effect.Effect = +export const makeDockerHelperRegistry = ( + ownerId: string, +): Effect.Effect => Effect.gen(function* () { const helpers = yield* Ref.make( new Map< @@ -86,5 +90,5 @@ export const makeDockerHelperRegistry: Effect.Effect const seen = yield* Ref.make>([]); yield* Effect.scoped( Effect.gen(function* () { - const registry = yield* makeDockerHelperRegistry; + const registry = yield* makeDockerHelperRegistry("owner-one"); const open = Ref.updateAndGet(opened, (count) => count + 1).pipe( Effect.map((count) => ({ id: `helper-${String(count)}`, created: true })), ); @@ -28,7 +28,7 @@ it.effect("opens a helper once and closes it with the host scope", () => it.effect("keeps the helper when a command fails", () => Effect.gen(function* () { const opened = yield* Ref.make(0); - const registry = yield* makeDockerHelperRegistry; + const registry = yield* makeDockerHelperRegistry("owner-one"); const open = Ref.updateAndGet(opened, (count) => count + 1).pipe( Effect.map((count) => ({ id: `helper-${String(count)}`, created: true })), ); @@ -48,7 +48,7 @@ it.effect("does not remove a helper this process did not create", () => const closed = yield* Ref.make>([]); yield* Effect.scoped( Effect.gen(function* () { - const registry = yield* makeDockerHelperRegistry; + const registry = yield* makeDockerHelperRegistry("owner-one"); yield* registry.use( "volume", Effect.succeed({ id: "adopted", created: false }), From 2f3d112c3be90fc2c900faa395485f10e57a8acb Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Thu, 24 Sep 2026 10:49:38 +0200 Subject: [PATCH 18/20] fix(stack): clean up interrupted helpers and finish destroy --- .../src/Orchestrator.integration.test.ts | 65 ++++++++++++++++++- packages/stack/src/Orchestrator.ts | 17 +++-- .../src/storage/DockerDatabaseStorage.ts | 59 ++++++++++------- .../stack/src/storage/DockerHelperRegistry.ts | 49 +++++++------- .../storage/DockerHelperRegistry.unit.test.ts | 37 ++++++----- 5 files changed, 156 insertions(+), 71 deletions(-) diff --git a/packages/stack/src/Orchestrator.integration.test.ts b/packages/stack/src/Orchestrator.integration.test.ts index 8bca0be8d1..90dc4f77d4 100644 --- a/packages/stack/src/Orchestrator.integration.test.ts +++ b/packages/stack/src/Orchestrator.integration.test.ts @@ -1,5 +1,18 @@ import { describe, expect, it } from "@effect/vitest"; -import { Context, Deferred, Effect, Exit, Fiber, Layer, Ref, Schema, Scope, Stream } from "effect"; +import { + Cause, + Context, + Deferred, + Effect, + Exit, + Fiber, + Layer, + Option, + Ref, + Schema, + Scope, + Stream, +} from "effect"; import * as TestClock from "effect/testing/TestClock"; import * as Orchestrator from "./Orchestrator.ts"; import type { RegisteredInstance } from "./Orchestrator.ts"; @@ -19,6 +32,7 @@ const makeInstance = ( readonly prepare?: Effect.Effect; readonly launch?: Effect.Effect; readonly stop?: Effect.Effect; + readonly removeData?: Effect.Effect; readonly exit?: Deferred.Deferred>; readonly events?: Ref.Ref>; } = {}, @@ -50,7 +64,7 @@ const makeInstance = ( remove: Effect.void, }; }), - removeData: () => Effect.void, + removeData: () => event("destroy").pipe(Effect.andThen(options.removeData ?? Effect.void)), }, { id, config: {}, coordinate: orchestrator.admissionFor(id) }, ); @@ -322,6 +336,53 @@ describe("service composition", () => { }), ).pipe(Effect.provide(TestClock.layer())), ); + + it.live("continues namespace destruction after a service data removal fails", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const events = yield* Ref.make>([]); + const failRemoval = yield* Ref.make(true); + const next = yield* makeInstance(orchestrator, "next", { events }); + const failing = yield* makeInstance(orchestrator, "failing", { + events, + removeData: Ref.get(failRemoval).pipe( + Effect.flatMap((fail) => + fail ? Effect.fail(failure("cannot remove data")) : Effect.void, + ), + ), + }); + yield* orchestrator.start("failing"); + yield* orchestrator.start("next"); + + const result = yield* Effect.exit(orchestrator.destroyNamespace); + expect(Exit.isFailure(result)).toBe(true); + if (Exit.isSuccess(result)) + return yield* Effect.die("namespace destroy unexpectedly passed"); + const error = Option.getOrUndefined(Cause.findErrorOption(result.cause)); + expect(error).toBeInstanceOf(Orchestrator.OrchestratorError); + if (!(error instanceof Orchestrator.OrchestratorError)) + return yield* Effect.die("namespace destroy did not return a combined error"); + expect(error.operation).toBe("destroy"); + expect( + error.outcomes?.map(({ id, result: outcome }) => [id, Exit.isSuccess(outcome)]), + ).toEqual([ + ["failing", false], + ["next", true], + ]); + expect((yield* Ref.get(events)).filter((event) => event.startsWith("destroy:"))).toEqual([ + "destroy:failing", + "destroy:next", + ]); + expect((yield* failing.core.get).registered).toBe(true); + expect((yield* next.core.get).registered).toBe(false); + + yield* Ref.set(failRemoval, false); + yield* orchestrator.destroyNamespace; + expect((yield* failing.core.get).registered).toBe(false); + }), + ), + ); }); it.live("joins an in-progress wake and wakes again after an admitted idle stop", () => diff --git a/packages/stack/src/Orchestrator.ts b/packages/stack/src/Orchestrator.ts index 00c51bb87b..94c156f5be 100644 --- a/packages/stack/src/Orchestrator.ts +++ b/packages/stack/src/Orchestrator.ts @@ -700,11 +700,20 @@ const makeOrchestrator = Effect.gen(function* () { }); const destroyNamespace = Effect.fn("Orchestrator.destroyNamespace")(function* () { const values = yield* Ref.get(registry); - // Each destroy stops that service. A prior waited stop would block ephemeral - // databases that only need the stop command to be issued. const configured = yield* configurationGraph(yield* Ref.get(composition)); - for (const id of topo([...values.keys()], configured.prerequisites).toReversed()) - yield* destroyNode(id); + const order = topo([...values.keys()], configured.prerequisites).toReversed(); + const outcomes = yield* Effect.forEach(order, (id) => + destroyNode(id).pipe( + Effect.exit, + Effect.map((result) => ({ id, result })), + ), + ); + if (outcomes.some(({ result }) => Exit.isFailure(result))) + return yield* new OrchestratorError({ + operation: "destroy", + message: "Composition destroy had failures", + outcomes, + }); }); const orchestrator: Interface = { diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index b935602497..8964bb4a01 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -633,8 +633,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") missingContainer(cause.message) ? Effect.succeed("absent") : Effect.fail(cause), ), ); - if (status === "running" || status === "restarting" || status === "paused") - return { id: name, created: false }; + if (status === "running" || status === "restarting" || status === "paused") return name; if (status === "created" || status === "exited" || status === "dead") { const removed = yield* engineCommand(["rm", name]).pipe( Effect.map(() => "removed"), @@ -646,33 +645,43 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") : Effect.fail(cause), ), ); - if (removed === "running") return { id: name, created: false }; + if (removed === "running") return name; } if (options.container === undefined) return yield* errorFor("helper", "Container runtime is unavailable"); yield* options.container.prepare(HELPER_IMAGE); - return yield* engineCommand([ - "run", - "-d", - "--name", - name, - "--label", - "com.supabase.stack-managed=true", - "--label", - "com.supabase.stack-helper=volume", - "--label", - `com.supabase.stack=${options.stackId}`, - ...mountArgs(mounts), - HELPER_IMAGE, - "/bin/sh", - "-c", - "trap : TERM INT; while :; do sleep 3600; done", - ]).pipe( - Effect.as({ id: name, created: true }), - Effect.catchTag("DockerDatabaseStorageError", (cause) => - /already in use/iu.test(cause.message) - ? Effect.succeed({ id: name, created: false }) - : Effect.fail(cause), + return yield* Effect.uninterruptibleMask((restore) => + restore( + engineCommand([ + "run", + "-d", + "--name", + name, + "--label", + "com.supabase.stack-managed=true", + "--label", + "com.supabase.stack-helper=volume", + "--label", + `com.supabase.stack=${options.stackId}`, + ...mountArgs(mounts), + HELPER_IMAGE, + "/bin/sh", + "-c", + "trap : TERM INT; while :; do sleep 3600; done", + ]).pipe( + Effect.as(name), + Effect.catchTag("DockerDatabaseStorageError", (cause) => + /already in use/iu.test(cause.message) + ? Effect.succeed(name) + : Effect.fail(cause), + ), + ), + ).pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) + ? closeSharedHelper(name).pipe(Effect.catch(Effect.logError)) + : Effect.void, + ), ), ); }); diff --git a/packages/stack/src/storage/DockerHelperRegistry.ts b/packages/stack/src/storage/DockerHelperRegistry.ts index 10baf5d071..a904f7e4c7 100644 --- a/packages/stack/src/storage/DockerHelperRegistry.ts +++ b/packages/stack/src/storage/DockerHelperRegistry.ts @@ -1,10 +1,4 @@ -import { Effect, Ref, Scope, Semaphore } from "effect"; - -interface DockerHelperHandle { - readonly id: string; - /** False when this process found a helper another process already created. */ - readonly created: boolean; -} +import { Cause, Effect, Exit, Ref, Scope, Semaphore } from "effect"; export interface DockerHelperRegistry { /** Identifies helpers owned by one host process. */ @@ -12,7 +6,7 @@ export interface DockerHelperRegistry { /** Runs `body` with the helper for `key`, opening it once per host lifetime. */ readonly use: ( key: string, - open: Effect.Effect, + open: Effect.Effect, close: (id: string) => Effect.Effect, body: (id: string) => Effect.Effect, ) => Effect.Effect; @@ -48,35 +42,40 @@ export const makeDockerHelperRegistry = ( ); const use = ( key: string, - open: Effect.Effect, + open: Effect.Effect, close: (id: string) => Effect.Effect, body: (id: string) => Effect.Effect, - ): Effect.Effect => - gate.withPermit( + ): Effect.Effect => { + const operation = gate.withPermit( Effect.gen(function* () { const current = yield* Ref.get(helpers); const existing = current.get(key); const id = existing?.id ?? - (yield* open.pipe( - Effect.tap((opened) => - Ref.update(helpers, (map) => { - const next = new Map(map); - next.set(key, { - id: opened.id, - close: (helperId) => - (opened.created ? close(helperId) : Effect.void).pipe( - Effect.catch((cause) => Effect.logError(cause)), - ), - }); - return next; - }), + (yield* Effect.uninterruptibleMask((restore) => + restore(open).pipe( + Effect.flatMap((id) => + Ref.update(helpers, (map) => { + const next = new Map(map); + next.set(key, { + id, + close: (helperId) => + close(helperId).pipe(Effect.catch((cause) => Effect.logError(cause))), + }); + return next; + }).pipe(Effect.as(id)), + ), ), - Effect.map((opened) => opened.id), )); return yield* body(id); }), ); + return operation.pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) && Cause.hasInterrupts(exit.cause) ? drop(key) : Effect.void, + ), + ); + }; const drop = (key: string): Effect.Effect => gate.withPermit( Effect.gen(function* () { diff --git a/packages/stack/src/storage/DockerHelperRegistry.unit.test.ts b/packages/stack/src/storage/DockerHelperRegistry.unit.test.ts index 460299d12b..3bce181c5d 100644 --- a/packages/stack/src/storage/DockerHelperRegistry.unit.test.ts +++ b/packages/stack/src/storage/DockerHelperRegistry.unit.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "@effect/vitest"; -import { Effect, Exit, Ref } from "effect"; +import { Deferred, Effect, Exit, Fiber, Ref } from "effect"; import { makeDockerHelperRegistry } from "./DockerHelperRegistry.ts"; it.effect("opens a helper once and closes it with the host scope", () => @@ -11,7 +11,7 @@ it.effect("opens a helper once and closes it with the host scope", () => Effect.gen(function* () { const registry = yield* makeDockerHelperRegistry("owner-one"); const open = Ref.updateAndGet(opened, (count) => count + 1).pipe( - Effect.map((count) => ({ id: `helper-${String(count)}`, created: true })), + Effect.map((count) => `helper-${String(count)}`), ); const close = (id: string) => Ref.update(closed, (ids) => [...ids, id]); yield* registry.use("volume", open, close, (id) => Ref.update(seen, (ids) => [...ids, id])); @@ -30,7 +30,7 @@ it.effect("keeps the helper when a command fails", () => const opened = yield* Ref.make(0); const registry = yield* makeDockerHelperRegistry("owner-one"); const open = Ref.updateAndGet(opened, (count) => count + 1).pipe( - Effect.map((count) => ({ id: `helper-${String(count)}`, created: true })), + Effect.map((count) => `helper-${String(count)}`), ); const close = (_id: string) => Effect.void; const failed = yield* registry @@ -43,20 +43,27 @@ it.effect("keeps the helper when a command fails", () => }), ); -it.effect("does not remove a helper this process did not create", () => +it.effect("closes an interrupted helper after releasing the registry gate", () => Effect.gen(function* () { + const opened = yield* Ref.make(0); const closed = yield* Ref.make>([]); - yield* Effect.scoped( - Effect.gen(function* () { - const registry = yield* makeDockerHelperRegistry("owner-one"); - yield* registry.use( - "volume", - Effect.succeed({ id: "adopted", created: false }), - (id) => Ref.update(closed, (ids) => [...ids, id]), - (id) => Effect.succeed(id), - ); - }), + const entered = yield* Deferred.make(); + const registry = yield* makeDockerHelperRegistry("owner-one"); + const open = Ref.updateAndGet(opened, (count) => count + 1).pipe( + Effect.map((count) => `helper-${String(count)}`), ); - expect(yield* Ref.get(closed)).toEqual([]); + const close = (id: string) => Ref.update(closed, (ids) => [...ids, id]); + const interrupted = yield* registry + .use("volume", open, close, () => + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never)), + ) + .pipe(Effect.forkChild); + yield* Deferred.await(entered); + const exit = yield* Fiber.interrupt(interrupted).pipe(Effect.andThen(Fiber.await(interrupted))); + expect(Exit.isFailure(exit)).toBe(true); + expect(yield* Ref.get(closed)).toEqual(["helper-1"]); + + const replacement = yield* registry.use("volume", open, close, (id) => Effect.succeed(id)); + expect(replacement).toBe("helper-2"); }), ); From fef2cb26c8051df624b2a3517f22dc242cf7161a Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Thu, 24 Sep 2026 10:54:50 +0200 Subject: [PATCH 19/20] fix(stack): retire only interrupted active helpers --- .../stack/src/storage/DockerHelperRegistry.ts | 39 ++++++++----------- .../storage/DockerHelperRegistry.unit.test.ts | 38 +++++++++++++++--- 2 files changed, 48 insertions(+), 29 deletions(-) diff --git a/packages/stack/src/storage/DockerHelperRegistry.ts b/packages/stack/src/storage/DockerHelperRegistry.ts index a904f7e4c7..79ede8af52 100644 --- a/packages/stack/src/storage/DockerHelperRegistry.ts +++ b/packages/stack/src/storage/DockerHelperRegistry.ts @@ -1,4 +1,4 @@ -import { Cause, Effect, Exit, Ref, Scope, Semaphore } from "effect"; +import { Effect, Ref, Scope, Semaphore } from "effect"; export interface DockerHelperRegistry { /** Identifies helpers owned by one host process. */ @@ -40,13 +40,24 @@ export const makeDockerHelperRegistry = ( }), ), ); + const forgetAndClose = (key: string): Effect.Effect => + Effect.gen(function* () { + const entry = (yield* Ref.get(helpers)).get(key); + if (entry === undefined) return; + yield* Ref.update(helpers, (map) => { + const next = new Map(map); + next.delete(key); + return next; + }); + yield* release(entry.id, entry.close); + }); const use = ( key: string, open: Effect.Effect, close: (id: string) => Effect.Effect, body: (id: string) => Effect.Effect, - ): Effect.Effect => { - const operation = gate.withPermit( + ): Effect.Effect => + gate.withPermit( Effect.gen(function* () { const current = yield* Ref.get(helpers); const existing = current.get(key); @@ -67,27 +78,9 @@ export const makeDockerHelperRegistry = ( ), ), )); - return yield* body(id); - }), - ); - return operation.pipe( - Effect.onExit((exit) => - Exit.isFailure(exit) && Cause.hasInterrupts(exit.cause) ? drop(key) : Effect.void, - ), - ); - }; - const drop = (key: string): Effect.Effect => - gate.withPermit( - Effect.gen(function* () { - const entry = (yield* Ref.get(helpers)).get(key); - if (entry === undefined) return; - yield* Ref.update(helpers, (map) => { - const next = new Map(map); - next.delete(key); - return next; - }); - yield* release(entry.id, entry.close); + return yield* body(id).pipe(Effect.onInterrupt(() => forgetAndClose(key))); }), ); + const drop = (key: string): Effect.Effect => gate.withPermit(forgetAndClose(key)); return { ownerId, use, drop }; }); diff --git a/packages/stack/src/storage/DockerHelperRegistry.unit.test.ts b/packages/stack/src/storage/DockerHelperRegistry.unit.test.ts index 3bce181c5d..adf48cda18 100644 --- a/packages/stack/src/storage/DockerHelperRegistry.unit.test.ts +++ b/packages/stack/src/storage/DockerHelperRegistry.unit.test.ts @@ -43,23 +43,49 @@ it.effect("keeps the helper when a command fails", () => }), ); -it.effect("closes an interrupted helper after releasing the registry gate", () => +it.effect("closes only when an interrupted use entered the helper body", () => Effect.gen(function* () { const opened = yield* Ref.make(0); const closed = yield* Ref.make>([]); - const entered = yield* Deferred.make(); + const enteredFirst = yield* Deferred.make(); + const releaseFirst = yield* Deferred.make(); + const waiterStarted = yield* Deferred.make(); const registry = yield* makeDockerHelperRegistry("owner-one"); const open = Ref.updateAndGet(opened, (count) => count + 1).pipe( Effect.map((count) => `helper-${String(count)}`), ); const close = (id: string) => Ref.update(closed, (ids) => [...ids, id]); - const interrupted = yield* registry + const first = yield* registry .use("volume", open, close, () => - Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never)), + Deferred.succeed(enteredFirst, undefined).pipe( + Effect.andThen(Deferred.await(releaseFirst)), + ), ) .pipe(Effect.forkChild); - yield* Deferred.await(entered); - const exit = yield* Fiber.interrupt(interrupted).pipe(Effect.andThen(Fiber.await(interrupted))); + yield* Deferred.await(enteredFirst); + const waiter = yield* Effect.gen(function* () { + yield* Deferred.succeed(waiterStarted, undefined); + return yield* registry.use("volume", open, close, () => Effect.succeed("unexpected")); + }).pipe(Effect.forkChild); + yield* Deferred.await(waiterStarted); + const waiterExit = yield* Fiber.interrupt(waiter).pipe(Effect.andThen(Fiber.await(waiter))); + expect(Exit.isFailure(waiterExit)).toBe(true); + expect(yield* Ref.get(closed)).toEqual([]); + yield* Deferred.succeed(releaseFirst, undefined); + yield* Fiber.await(first); + + const reused = yield* registry.use("volume", open, close, (id) => Effect.succeed(id)); + expect(reused).toBe("helper-1"); + expect(yield* Ref.get(opened)).toBe(1); + + const enteredActive = yield* Deferred.make(); + const active = yield* registry + .use("volume", open, close, () => + Deferred.succeed(enteredActive, undefined).pipe(Effect.andThen(Effect.never)), + ) + .pipe(Effect.forkChild); + yield* Deferred.await(enteredActive); + const exit = yield* Fiber.interrupt(active).pipe(Effect.andThen(Fiber.await(active))); expect(Exit.isFailure(exit)).toBe(true); expect(yield* Ref.get(closed)).toEqual(["helper-1"]); From 0eaf4280c4173062a930883f43311fe09666bad3 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Thu, 24 Sep 2026 13:07:35 +0200 Subject: [PATCH 20/20] fix(stack): launch storage helpers from prepared image --- packages/stack/src/runtime/Container.ts | 21 ++- .../DockerDatabaseStorage.integration.test.ts | 156 ++++++++++++++++++ .../src/storage/DockerDatabaseStorage.ts | 12 +- 3 files changed, 179 insertions(+), 10 deletions(-) diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index b332577e71..ceb0a637e9 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -65,6 +65,7 @@ export class ContainerLaunchError extends Data.TaggedError("ContainerLaunchError export interface ContainerRuntime { readonly prepare: (image: string) => Effect.Effect; + readonly prepareImage: (image: string) => Effect.Effect; readonly launch: ( spec: ContainerSpec, ) => Effect.Effect; @@ -169,12 +170,13 @@ export const makeContainerRuntime = (options: { image: string, mirrors: ReadonlyArray, primaryError: ContainerError, - ): Effect.Effect => { + ): Effect.Effect => { const [mirror, ...rest] = mirrors; if (mirror === undefined) return Effect.fail(primaryError); return Effect.gen(function* () { if (!(yield* present(mirror))) yield* pull(mirror); yield* Ref.update(mirrored, (map) => new Map(map).set(image, mirror)); + return mirror; }).pipe( Effect.tap(() => Effect.logInfo(`Pulled image from mirror ${mirror}`)), Effect.tapError((cause) => Effect.logWarning(`Image mirror ${mirror} failed`, cause)), @@ -182,7 +184,7 @@ export const makeContainerRuntime = (options: { ); }; - const prepare = Effect.fn("Container.prepare")(function* (image: string) { + const prepareImage = Effect.fn("Container.prepareImage")(function* (image: string) { // A mirror chosen earlier may have been pruned since; launches follow the primary again. const usePrimary = Ref.update(mirrored, (map) => { if (!map.has(image)) return map; @@ -190,13 +192,20 @@ export const makeContainerRuntime = (options: { next.delete(image); return next; }); - if (yield* present(image)) return yield* usePrimary; + if (yield* present(image)) { + yield* usePrimary; + return image; + } const mirrors = options.imageMirrors?.(image) ?? []; - yield* pull(image).pipe( - Effect.andThen(usePrimary), + return yield* pull(image).pipe( + Effect.as(image), + Effect.tap(() => usePrimary), Effect.catch((primaryError) => fromMirror(image, mirrors, primaryError)), ); }); + const prepare = Effect.fn("Container.prepare")((image: string) => + prepareImage(image).pipe(Effect.asVoid), + ); const launch = Effect.fn("Container.launch")(function* ( spec: ContainerSpec, @@ -422,5 +431,5 @@ export const makeContainerRuntime = (options: { }), ); }); - return { prepare, launch, launchTool: (spec) => launch(spec, true) }; + return { prepare, prepareImage, launch, launchTool: (spec) => launch(spec, true) }; }); diff --git a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts index c2ca4f2d08..763a771dbb 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts @@ -6,9 +6,11 @@ import { Effect, Exit, FileSystem, + Layer, Option, Path, Schema, + Sink, Scope, Stream, } from "effect"; @@ -37,6 +39,50 @@ class DockerTestError extends Data.TaggedError("DockerTestError")<{ readonly message: string; }> {} +const helperMirror = "registry.test/supabase/postgres:17"; +const fakeHelperEngine = () => { + const commands: string[][] = []; + const local = new Set(); + const handle = (exitCode: number, stdout = "", stderr = "") => + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(4242), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(exitCode)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + stdout: Stream.succeed(new TextEncoder().encode(stdout)), + stderr: Stream.succeed(new TextEncoder().encode(stderr)), + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + const spawner = ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command)) return Effect.die("unexpected piped command"); + const args = [...command.args]; + commands.push(args); + if (args[0] === "image") + return Effect.succeed(handle(0, local.has(args.at(-1) ?? "") ? "sha256:1" : "")); + if (args[0] === "pull") { + if (args.at(-1) !== helperMirror) + return Effect.succeed(handle(1, "", `denied: ${args.at(-1)}`)); + local.add(helperMirror); + return Effect.succeed(handle(0)); + } + if (args[0] === "inspect") return Effect.succeed(handle(1, "", "no such container")); + if (args[0] === "run") { + const shellIndex = args.indexOf("/bin/sh"); + const image = args[shellIndex - 1]; + return image !== undefined && local.has(image) + ? Effect.succeed(handle(0, "abcdef0123456789")) + : Effect.succeed(handle(1, "", `Unable to find image '${image ?? ""}' locally`)); + } + if (args[0] === "exec" || args[0] === "rm") return Effect.succeed(handle(0, "done")); + return Effect.succeed(handle(1, "", `unexpected engine command: ${args[0] ?? ""}`)); + }); + return { commands, layer: Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner) }; +}; + const docker = Effect.fn("DockerStorageTest.docker")((args: ReadonlyArray) => Effect.scoped( Effect.gen(function* () { @@ -74,6 +120,116 @@ const docker = Effect.fn("DockerStorageTest.docker")((args: ReadonlyArray `'${value.replaceAll("'", "'\\''")}'`; describe("Docker database storage", { timeout: 120_000 }, () => { + it.live("starts a storage helper with the mirror image selected during preparation", () => { + const engine = fakeHelperEngine(); + return Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "storage-helper-mirror-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "database"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + engine: "podman", + imageMirrors: () => [helperMirror], + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const storage = yield* makeDockerDatabaseStorage({ + runtime: "podman", + stackId: "storage-helper-mirror", + instanceId: "database", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + + yield* storage.prepare("17"); + yield* storage.removeData("17"); + + const run = engine.commands.find((args) => args[0] === "run"); + const shellIndex = run?.indexOf("/bin/sh") ?? -1; + expect(run?.[shellIndex - 1]).toBe(helperMirror); + expect( + engine.commands.filter((args) => args[0] === "pull").map((args) => args.at(-1)), + ).toEqual([expect.stringContaining("ghcr.io/supabase/cli/postgres:"), helperMirror]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); + + it.live("starts a shared volume helper with the mirror image selected during preparation", () => { + const engine = fakeHelperEngine(); + return Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "storage-shared-helper-mirror-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "database"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const stackId = "shared-helper-mirror"; + const instanceId = "database"; + const cachePath = yield* fs.realPath(cacheRoot); + const cacheHash = yield* crypto.digest("SHA-256", new TextEncoder().encode(cachePath)); + const cacheNamespace = `cache-${Array.from(cacheHash, (byte) => + byte.toString(16).padStart(2, "0"), + ) + .join("") + .slice(0, 32)}`; + const marker = yield* Schema.encodeEffect(Schema.fromJsonString(Marker))({ + backend: "docker", + volume: "database-volume", + namespace: `instance-${stackId}-${instanceId}`, + cacheNamespace, + initialized: false, + }); + yield* fs.writeFileString( + path.join(instanceRoot, ".supabase-database-storage.json"), + marker, + ); + const container = yield* makeContainerRuntime({ + engine: "podman", + imageMirrors: () => [helperMirror], + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const helpers = yield* makeDockerHelperRegistry("mirror-test"); + const storage = yield* makeDockerDatabaseStorage({ + runtime: "podman", + stackId, + instanceId, + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + helpers, + }); + + yield* storage.prepare("17"); + + const run = engine.commands.find((args) => args[0] === "run"); + const shellIndex = run?.indexOf("/bin/sh") ?? -1; + expect(run?.[shellIndex - 1]).toBe(helperMirror); + expect( + engine.commands.filter((args) => args[0] === "pull").map((args) => args.at(-1)), + ).toEqual([expect.stringContaining("ghcr.io/supabase/cli/postgres:"), helperMirror]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); + it.live("round-trips PostgreSQL 15 data with its catalog image", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index 4cea61d7e5..a4b929f3d6 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -579,7 +579,9 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* options.fs .makeDirectory(options.cacheRoot, { recursive: true }) .pipe(Effect.mapError((cause) => errorFor("helper", cause))); - yield* options.container.prepare(image); + const preparedImage = yield* options.container + .prepareImage(image) + .pipe(Effect.mapError((cause) => errorFor("helper", cause))); const token = yield* options.crypto.randomUUIDv4.pipe( Effect.mapError((cause) => errorFor("helper", cause)), ); @@ -600,7 +602,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") "--label", `com.supabase.instance=${options.instanceId}`, ...mountArgs(mounts), - image, + preparedImage, "/bin/sh", "-c", "trap : TERM INT; while :; do sleep 3600; done", @@ -670,7 +672,9 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") } if (options.container === undefined) return yield* errorFor("helper", "Container runtime is unavailable"); - yield* options.container.prepare(image); + const preparedImage = yield* options.container + .prepareImage(image) + .pipe(Effect.mapError((cause) => errorFor("helper", cause))); return yield* Effect.uninterruptibleMask((restore) => restore( engineCommand([ @@ -685,7 +689,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") "--label", `com.supabase.stack=${options.stackId}`, ...mountArgs(mounts), - image, + preparedImage, "/bin/sh", "-c", "trap : TERM INT; while :; do sleep 3600; done",