diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index 50ba106eaf..c28f4bc3b4 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -72,12 +72,12 @@ const definitions: Readonly> = { database: definition( "postgres", "17.6.1.173", - "ghcr.io/supabase/cli/postgres:17.6.1.173@sha256:1581c433d71a48a81e356a3ed2d4aa5ecfc8fc0465ea98661da7a88023317dcf", + "ghcr.io/supabase/cli/postgres:17.6.1.173@sha256:9d6e542382946cad5eb1f11f1c8108a51297902ee42fe5098358816d3784ba5a", "bin/supabase-postgres-start", ["bin/supabase-postgres-start", "bin/pg_dump", "bin/pg_dumpall", "bin/pg_prove", "bin/psql"], { "15.14.1.173": - "ghcr.io/supabase/cli/postgres:15.14.1.173@sha256:b7d210fa3bca26568fa20448e3bec092bd4f8da6a5af5029597639d38e7d896e", + "ghcr.io/supabase/cli/postgres:15.14.1.173@sha256:3abb20e89700d6211e741148b85c3a052bb290cae3b4f751311442f4a5fe2324", }, ), rest: definition("postgrest", "v16.2", "ghcr.io/supabase/cli/postgrest:v16.2", "bin/postgrest"), diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index b332577e71..ceb0a637e9 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -65,6 +65,7 @@ export class ContainerLaunchError extends Data.TaggedError("ContainerLaunchError export interface ContainerRuntime { readonly prepare: (image: string) => Effect.Effect; + readonly prepareImage: (image: string) => Effect.Effect; readonly launch: ( spec: ContainerSpec, ) => Effect.Effect; @@ -169,12 +170,13 @@ export const makeContainerRuntime = (options: { image: string, mirrors: ReadonlyArray, primaryError: ContainerError, - ): Effect.Effect => { + ): Effect.Effect => { const [mirror, ...rest] = mirrors; if (mirror === undefined) return Effect.fail(primaryError); return Effect.gen(function* () { if (!(yield* present(mirror))) yield* pull(mirror); yield* Ref.update(mirrored, (map) => new Map(map).set(image, mirror)); + return mirror; }).pipe( Effect.tap(() => Effect.logInfo(`Pulled image from mirror ${mirror}`)), Effect.tapError((cause) => Effect.logWarning(`Image mirror ${mirror} failed`, cause)), @@ -182,7 +184,7 @@ export const makeContainerRuntime = (options: { ); }; - const prepare = Effect.fn("Container.prepare")(function* (image: string) { + const prepareImage = Effect.fn("Container.prepareImage")(function* (image: string) { // A mirror chosen earlier may have been pruned since; launches follow the primary again. const usePrimary = Ref.update(mirrored, (map) => { if (!map.has(image)) return map; @@ -190,13 +192,20 @@ export const makeContainerRuntime = (options: { next.delete(image); return next; }); - if (yield* present(image)) return yield* usePrimary; + if (yield* present(image)) { + yield* usePrimary; + return image; + } const mirrors = options.imageMirrors?.(image) ?? []; - yield* pull(image).pipe( - Effect.andThen(usePrimary), + return yield* pull(image).pipe( + Effect.as(image), + Effect.tap(() => usePrimary), Effect.catch((primaryError) => fromMirror(image, mirrors, primaryError)), ); }); + const prepare = Effect.fn("Container.prepare")((image: string) => + prepareImage(image).pipe(Effect.asVoid), + ); const launch = Effect.fn("Container.launch")(function* ( spec: ContainerSpec, @@ -422,5 +431,5 @@ export const makeContainerRuntime = (options: { }), ); }); - return { prepare, launch, launchTool: (spec) => launch(spec, true) }; + return { prepare, prepareImage, launch, launchTool: (spec) => launch(spec, true) }; }); diff --git a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts index 17dbd21003..763a771dbb 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts @@ -6,21 +6,26 @@ import { Effect, Exit, FileSystem, + Layer, Option, Path, Schema, + Sink, Scope, Stream, } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { postgresVersion, resolveArtifact } from "../Artifacts.ts"; import { makeContainerRuntime } from "../runtime/Container.ts"; import { makeDatabaseSnapshots } from "../services/DatabaseSnapshot.ts"; import { makeDockerDatabaseStorage } from "./DockerDatabaseStorage.ts"; import { makeDockerHelperRegistry } from "./DockerHelperRegistry.ts"; import type { DockerHelperRegistry } from "./DockerHelperRegistry.ts"; -const helperImage = - "public.ecr.aws/docker/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; +const postgresImage = (version: string) => + resolveArtifact({ service: "database", version: postgresVersion(version) }).pipe( + Effect.map(({ image }) => image), + ); const Marker = Schema.Struct({ backend: Schema.Literals(["docker", "host"]), volume: Schema.optionalKey(Schema.String), @@ -34,6 +39,50 @@ class DockerTestError extends Data.TaggedError("DockerTestError")<{ readonly message: string; }> {} +const helperMirror = "registry.test/supabase/postgres:17"; +const fakeHelperEngine = () => { + const commands: string[][] = []; + const local = new Set(); + const handle = (exitCode: number, stdout = "", stderr = "") => + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(4242), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(exitCode)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + stdout: Stream.succeed(new TextEncoder().encode(stdout)), + stderr: Stream.succeed(new TextEncoder().encode(stderr)), + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + const spawner = ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command)) return Effect.die("unexpected piped command"); + const args = [...command.args]; + commands.push(args); + if (args[0] === "image") + return Effect.succeed(handle(0, local.has(args.at(-1) ?? "") ? "sha256:1" : "")); + if (args[0] === "pull") { + if (args.at(-1) !== helperMirror) + return Effect.succeed(handle(1, "", `denied: ${args.at(-1)}`)); + local.add(helperMirror); + return Effect.succeed(handle(0)); + } + if (args[0] === "inspect") return Effect.succeed(handle(1, "", "no such container")); + if (args[0] === "run") { + const shellIndex = args.indexOf("/bin/sh"); + const image = args[shellIndex - 1]; + return image !== undefined && local.has(image) + ? Effect.succeed(handle(0, "abcdef0123456789")) + : Effect.succeed(handle(1, "", `Unable to find image '${image ?? ""}' locally`)); + } + if (args[0] === "exec" || args[0] === "rm") return Effect.succeed(handle(0, "done")); + return Effect.succeed(handle(1, "", `unexpected engine command: ${args[0] ?? ""}`)); + }); + return { commands, layer: Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner) }; +}; + const docker = Effect.fn("DockerStorageTest.docker")((args: ReadonlyArray) => Effect.scoped( Effect.gen(function* () { @@ -71,12 +120,225 @@ const docker = Effect.fn("DockerStorageTest.docker")((args: ReadonlyArray `'${value.replaceAll("'", "'\\''")}'`; describe("Docker database storage", { timeout: 120_000 }, () => { + it.live("starts a storage helper with the mirror image selected during preparation", () => { + const engine = fakeHelperEngine(); + return Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "storage-helper-mirror-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "database"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + engine: "podman", + imageMirrors: () => [helperMirror], + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const storage = yield* makeDockerDatabaseStorage({ + runtime: "podman", + stackId: "storage-helper-mirror", + instanceId: "database", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + + yield* storage.prepare("17"); + yield* storage.removeData("17"); + + const run = engine.commands.find((args) => args[0] === "run"); + const shellIndex = run?.indexOf("/bin/sh") ?? -1; + expect(run?.[shellIndex - 1]).toBe(helperMirror); + expect( + engine.commands.filter((args) => args[0] === "pull").map((args) => args.at(-1)), + ).toEqual([expect.stringContaining("ghcr.io/supabase/cli/postgres:"), helperMirror]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); + + it.live("starts a shared volume helper with the mirror image selected during preparation", () => { + const engine = fakeHelperEngine(); + return Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "storage-shared-helper-mirror-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "database"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const stackId = "shared-helper-mirror"; + const instanceId = "database"; + const cachePath = yield* fs.realPath(cacheRoot); + const cacheHash = yield* crypto.digest("SHA-256", new TextEncoder().encode(cachePath)); + const cacheNamespace = `cache-${Array.from(cacheHash, (byte) => + byte.toString(16).padStart(2, "0"), + ) + .join("") + .slice(0, 32)}`; + const marker = yield* Schema.encodeEffect(Schema.fromJsonString(Marker))({ + backend: "docker", + volume: "database-volume", + namespace: `instance-${stackId}-${instanceId}`, + cacheNamespace, + initialized: false, + }); + yield* fs.writeFileString( + path.join(instanceRoot, ".supabase-database-storage.json"), + marker, + ); + const container = yield* makeContainerRuntime({ + engine: "podman", + imageMirrors: () => [helperMirror], + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const helpers = yield* makeDockerHelperRegistry("mirror-test"); + const storage = yield* makeDockerDatabaseStorage({ + runtime: "podman", + stackId, + instanceId, + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + helpers, + }); + + yield* storage.prepare("17"); + + const run = engine.commands.find((args) => args[0] === "run"); + const shellIndex = run?.indexOf("/bin/sh") ?? -1; + expect(run?.[shellIndex - 1]).toBe(helperMirror); + expect( + engine.commands.filter((args) => args[0] === "pull").map((args) => args.at(-1)), + ).toEqual([expect.stringContaining("ghcr.io/supabase/cli/postgres:"), helperMirror]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); + + it.live("round-trips PostgreSQL 15 data with its catalog image", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("15"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-pg15-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const sourceRoot = path.join(storageRoot, "source"); + const targetRoot = path.join(storageRoot, "target"); + yield* fs.makeDirectory(sourceRoot, { recursive: true }); + yield* fs.makeDirectory(targetRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ engine: "docker" }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-pg15-${yield* crypto.randomUUIDv4}`; + const makeStorage = (instanceId: string, instanceRoot: string) => + makeDockerDatabaseStorage({ + runtime: "docker", + stackId, + instanceId, + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + const source = yield* makeStorage("source", sourceRoot); + const target = yield* makeStorage("target", targetRoot); + const ownerScope = yield* Scope.Scope; + yield* Scope.addFinalizer( + ownerScope, + Effect.gen(function* () { + yield* source.destroyData("15").pipe(Effect.ignore); + yield* target.destroyData("15").pipe(Effect.ignore); + const marker = yield* fs + .readFileString(path.join(sourceRoot, ".supabase-database-storage.json")) + .pipe(Effect.option); + if (Option.isSome(marker)) { + const parsed = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + marker.value, + ).pipe(Effect.option); + if (Option.isSome(parsed) && parsed.value.volume !== undefined) + yield* docker(["volume", "rm", parsed.value.volume]).pipe(Effect.ignore); + } + }).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.ignore, + ), + ); + + yield* source.prepare("15"); + const sourceMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), + ); + if (sourceMarker.backend !== "docker" || sourceMarker.volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `set -eu; printf 15 > ${quote(`/store/${sourceMarker.namespace}/data/PG_VERSION`)}; printf pg15-source > ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}`, + ]); + yield* source.markInitialized("15"); + yield* fs.writeFileString( + path.join(sourceRoot, ".supabase-database-ready.json"), + '{"version":"15","runtime":"docker","profile":"supabase"}', + ); + yield* source.saveSnapshot("15", "restore-me"); + + yield* target.prepare("15"); + expect(yield* target.restoreSnapshot("15", "restore-me")).toBe(true); + const targetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(targetRoot, ".supabase-database-storage.json")), + ); + expect(targetMarker.volume).toBe(sourceMarker.volume); + yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store,volume-subpath=${targetMarker.namespace}/data`, + helperImage, + "/bin/sh", + "-c", + 'test "$(cat /store/PG_VERSION)" = 15 && test "$(cat /store/fixture)" = pg15-source', + ]); + yield* target.destroyData("15"); + yield* source.destroyData("15"); + yield* docker(["volume", "rm", sourceMarker.volume]); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + it.live("round-trips stopped data and protects managed namespaces", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-" }); const stateRoot = path.join(root, "state"); const storageRoot = path.join(stateRoot, "stack", "data"); @@ -146,7 +408,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { helperImage, "/bin/sh", "-c", - `set -eu; printf 17 > ${quote(`/store/${sourceMarker.namespace}/data/PG_VERSION`)}; printf source > ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}`, + `set -eu; printf 17 > ${quote(`/store/${sourceMarker.namespace}/data/PG_VERSION`)}; printf source > ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}; /usr/bin/busybox setfattr -n user.storage-smoke -v preserved ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}`, ]); yield* source.markInitialized("17"); yield* fs.writeFileString( @@ -201,10 +463,10 @@ describe("Docker database storage", { timeout: 120_000 }, () => { helperImage, "/bin/sh", "-c", - `test "$(cat /store/fixture)" = source`, + `set -eu; test "$(cat /store/fixture)" = source; /usr/bin/busybox setfattr -x user.storage-smoke /store/fixture`, ]); - yield* target.removeData("17"); + yield* target.removeData("unsupported"); const resetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( yield* fs.readFileString(path.join(targetRoot, ".supabase-database-storage.json")), ); @@ -219,7 +481,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { helperImage, "/bin/sh", "-c", - 'descriptor=$(find /store -name descriptor.json -print -quit); printf corrupt > "$descriptor"', + 'descriptor=$(/usr/bin/busybox find /store -name descriptor.json -print -quit); printf corrupt > "$descriptor"', ]); const corrupt = yield* target.restoreSnapshot("17", "roundtrip").pipe( Effect.matchEffect({ @@ -245,6 +507,21 @@ describe("Docker database storage", { timeout: 120_000 }, () => { expect(missing).toSatisfy((exit) => Exit.isFailure(exit)); yield* target.removeData("17"); + yield* source.saveSnapshot("17", "retention-first"); + yield* source.saveSnapshot("17", "retention-second"); + yield* source.saveSnapshot("17", "retention-third"); + yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `test "$(/usr/bin/busybox find ${quote(`/store/${sourceMarker.cacheNamespace}/entries`)} -mindepth 1 -maxdepth 1 -type d | /usr/bin/busybox wc -l)" -eq 3`, + ]); + expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(false); + yield* source.saveSnapshot("17", "roundtrip"); yield* source.destroyData("17"); expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(true); yield* target.destroyData("17"); @@ -290,6 +567,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-legacy-" }); const storageRoot = path.join(root, "state", "stack", "data"); const cacheRoot = path.join(root, "cache"); @@ -498,6 +776,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-missing-" }); const storageRoot = path.join(root, "state", "stack", "data"); const cacheRoot = path.join(root, "cache"); @@ -576,6 +855,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-host-" }); const stateRoot = path.join(root, "state"); const storageRoot = path.join(stateRoot, "stack", "data"); @@ -674,7 +954,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { ]), ).toBe(expectedOwnership); yield* storage.removeData("17"); - yield* storage.destroyData("17"); + yield* storage.destroyData("unsupported"); expect(yield* fs.exists(path.join(instanceRoot, ".supabase-database-storage.json"))).toBe( true, ); @@ -690,6 +970,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-reopen-" }); const stateRoot = path.join(root, "state"); const storageRoot = path.join(stateRoot, "stack", "data"); diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index 8964bb4a01..a4b929f3d6 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -13,13 +13,11 @@ import { } from "effect"; import { ChildProcess } from "effect/unstable/process"; import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; +import { postgresVersion, resolveArtifact } from "../Artifacts.ts"; import type { ContainerRuntime } from "../runtime/Container.ts"; import type { DatabaseRuntime } from "../services/Database.ts"; import type { DockerHelperRegistry } from "./DockerHelperRegistry.ts"; -const HELPER_IMAGE = - "public.ecr.aws/docker/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; - const Marker = Schema.Struct({ backend: Schema.Literals(["docker", "host"]), volume: Schema.optionalKey(Schema.String), @@ -111,6 +109,9 @@ const errorFor = (operation: string, cause: unknown) => const shellQuote = (value: string): string => `'${value.replaceAll("'", "'\\''")}'`; +const withSnapshotLock = (root: string, command: string): string => + `set -eu; /usr/bin/busybox mkdir -p ${shellQuote(root)}; exec 9>${shellQuote(`${root}/.lock`)}; attempt=0; until lock_error=$(/usr/bin/busybox flock -n 9 2>&1); do if [ -n "$lock_error" ]; then echo "$lock_error" >&2; exit 1; fi; if [ "$attempt" -ge 120 ]; then echo 'Timed out waiting for database snapshot lock' >&2; exit 1; fi; attempt=$((attempt + 1)); /usr/bin/busybox sleep 1; done; /usr/bin/sh -eu -c ${shellQuote(command)}`; + const parseMajor = (version: string): number | undefined => { const major = Number(version.trim().split(".")[0]); return Number.isInteger(major) ? major : undefined; @@ -504,6 +505,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") }); const helperId = yield* Ref.make(undefined); + const helperImage = yield* Ref.make(undefined); const helperCleanupPending = yield* Ref.make(false); const operationLock = yield* Semaphore.make(1); const ownerScope = yield* Scope.Scope; @@ -536,6 +538,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ), ); yield* Ref.set(helperId, undefined); + yield* Ref.set(helperImage, undefined); yield* Ref.set(helperCleanupPending, false); } }), @@ -545,11 +548,24 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ownerScope, removeHelper().pipe(Effect.catch((cause) => Effect.logError(cause))), ); + const imageForVersion = (version: string, fallback: boolean) => + resolveArtifact({ service: "database", version: postgresVersion(version) }).pipe( + Effect.map(({ image }) => image), + Effect.catchTag("ArtifactError", (cause) => + fallback + ? resolveArtifact({ service: "database" }).pipe(Effect.map(({ image }) => image)) + : Effect.fail(cause), + ), + Effect.mapError((cause) => errorFor("helper", cause)), + ); const acquireHelper = Effect.fn("DockerDatabaseStorage.acquireHelper")( - (mounts: ReadonlyArray) => + (mounts: ReadonlyArray, image: string) => Effect.gen(function* () { const current = yield* Ref.get(helperId); - if (current !== undefined && (yield* Ref.get(helperCleanupPending))) + if ( + current !== undefined && + ((yield* Ref.get(helperCleanupPending)) || (yield* Ref.get(helperImage)) !== image) + ) yield* removeHelper(); const active = yield* Ref.get(helperId); if (active !== undefined) return active; @@ -563,7 +579,9 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* options.fs .makeDirectory(options.cacheRoot, { recursive: true }) .pipe(Effect.mapError((cause) => errorFor("helper", cause))); - yield* options.container.prepare(HELPER_IMAGE); + const preparedImage = yield* options.container + .prepareImage(image) + .pipe(Effect.mapError((cause) => errorFor("helper", cause))); const token = yield* options.crypto.randomUUIDv4.pipe( Effect.mapError((cause) => errorFor("helper", cause)), ); @@ -571,6 +589,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") // Register the deterministic owned name before the remote create starts so an // interrupted docker run can still be removed by the same scope. yield* Ref.set(helperId, name); + yield* Ref.set(helperImage, image); const created = yield* engineCommand([ "run", "-d", @@ -583,7 +602,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") "--label", `com.supabase.instance=${options.instanceId}`, ...mountArgs(mounts), - HELPER_IMAGE, + preparedImage, "/bin/sh", "-c", "trap : TERM INT; while :; do sleep 3600; done", @@ -595,17 +614,20 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ); const missingContainer = (message: string) => /no such (?:container|object)/iu.test(message); - const volumeHelperKey = (mounts: ReadonlyArray): string | undefined => { + const volumeHelperKey = ( + mounts: ReadonlyArray, + image: string, + ): string | undefined => { if (options.helpers === undefined) return undefined; if (mounts.length === 0 || mounts.some((mount) => (mount.type ?? "bind") !== "volume")) return undefined; - return mounts + return `${image}\0${mounts .map( (mount) => `${mount.source}\0${mount.target}\0${mount.volumeSubpath ?? ""}\0${mount.readOnly ? "ro" : "rw"}`, ) .sort() - .join("\n"); + .join("\n")}`; }; const closeSharedHelper = (id: string) => engineCommand(["rm", "-f", id]).pipe( @@ -618,6 +640,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") const openSharedHelper = ( mounts: ReadonlyArray, key: string, + image: string, ownerId: string, ) => Effect.gen(function* () { @@ -649,7 +672,9 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") } if (options.container === undefined) return yield* errorFor("helper", "Container runtime is unavailable"); - yield* options.container.prepare(HELPER_IMAGE); + const preparedImage = yield* options.container + .prepareImage(image) + .pipe(Effect.mapError((cause) => errorFor("helper", cause))); return yield* Effect.uninterruptibleMask((restore) => restore( engineCommand([ @@ -664,7 +689,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") "--label", `com.supabase.stack=${options.stackId}`, ...mountArgs(mounts), - HELPER_IMAGE, + preparedImage, "/bin/sh", "-c", "trap : TERM INT; while :; do sleep 3600; done", @@ -688,44 +713,50 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") const runHelper = Effect.fn("DockerDatabaseStorage.helper")(( command: string, mounts: ReadonlyArray, + version: string, + fallback = false, ): Effect.Effect => { - const sharedKey = volumeHelperKey(mounts); - if (sharedKey !== undefined && options.helpers !== undefined) { - const helpers = options.helpers; - const exec = (id: string) => engineCommand(["exec", id, "/bin/sh", "-c", command]); - const open = openSharedHelper(mounts, sharedKey, helpers.ownerId); - return helpers.use(sharedKey, open, closeSharedHelper, exec).pipe( - Effect.catchTag("DockerDatabaseStorageError", (cause) => - /no such container|is not running/iu.test(cause.message) - ? helpers - .drop(sharedKey) - .pipe(Effect.andThen(helpers.use(sharedKey, open, closeSharedHelper, exec))) - : Effect.fail(cause), - ), - Effect.mapError((cause) => errorFor("helper", cause)), - ); - } const cleanupAfterFailure = Effect.gen(function* () { if ((yield* Ref.get(helperId)) === undefined) return; yield* Ref.set(helperCleanupPending, true); yield* removeHelper(); }); - const operation = Effect.uninterruptibleMask((restore) => - restore( - Effect.gen(function* () { - const id = yield* acquireHelper(mounts); - return yield* engineCommand(["exec", id, "/bin/sh", "-c", command]); - }), - ).pipe( - Effect.onExit((exit) => - Exit.isFailure(exit) ? Effect.uninterruptible(cleanupAfterFailure) : Effect.void, - ), - Effect.mapError((cause) => errorFor("mount", cause)), - ), + return imageForVersion(version, fallback).pipe( + Effect.flatMap((image) => { + const sharedKey = volumeHelperKey(mounts, image); + if (sharedKey !== undefined && options.helpers !== undefined) { + const helpers = options.helpers; + const exec = (id: string) => engineCommand(["exec", id, "/bin/sh", "-c", command]); + const open = openSharedHelper(mounts, sharedKey, image, helpers.ownerId); + return helpers.use(sharedKey, open, closeSharedHelper, exec).pipe( + Effect.catchTag("DockerDatabaseStorageError", (cause) => + /no such container|is not running/iu.test(cause.message) + ? helpers + .drop(sharedKey) + .pipe(Effect.andThen(helpers.use(sharedKey, open, closeSharedHelper, exec))) + : Effect.fail(cause), + ), + Effect.mapError((cause) => errorFor("helper", cause)), + ); + } + const operation = Effect.uninterruptibleMask((restore) => + restore( + Effect.gen(function* () { + const id = yield* acquireHelper(mounts, image); + return yield* engineCommand(["exec", id, "/bin/sh", "-c", command]); + }), + ).pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) ? Effect.uninterruptible(cleanupAfterFailure) : Effect.void, + ), + Effect.mapError((cause) => errorFor("mount", cause)), + ), + ); + return operationLock + .withPermit(operation) + .pipe(Effect.mapError((cause) => errorFor("helper", cause))); + }), ); - return operationLock - .withPermit(operation) - .pipe(Effect.mapError((cause) => errorFor("helper", cause))); }); const getMarker = Effect.gen(function* () { @@ -801,6 +832,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") { source: options.instanceRoot, target: "/instance", readOnly: false }, { source: options.cacheRoot, target: "/cache", readOnly: false }, ], + version, ); if (major.trim() !== majorVersion(version)) return yield* errorFor( @@ -818,6 +850,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") const major = yield* runHelper( `set -eu; if [ ! -f ${shellQuote(`${store}/data/PG_VERSION`)} ]; then echo 'Initialized PostgreSQL data is missing PG_VERSION' >&2; exit 1; fi; cat ${shellQuote(`${store}/data/PG_VERSION`)}`, [{ source: marker.volume ?? "", target: "/store", readOnly: false, type: "volume" }], + version, ); if (major.trim() !== majorVersion(version)) return yield* errorFor( @@ -828,6 +861,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* runHelper( `set -eu; mkdir -p ${shellQuote(`${store}/data`)} ${shellQuote(`${cache}/entries`)} ${shellQuote(`${cache}/stages`)}; chown -R 100:101 ${shellQuote(store)}`, [{ source: marker.volume ?? "", target: "/store", readOnly: false, type: "volume" }], + version, ); } }).pipe(Effect.mapError((cause) => errorFor("prepare", cause))), @@ -867,12 +901,13 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* runHelper( `set -eu; if [ ! -f ${shellQuote(versionFile)} ]; then echo 'Database readiness requires PG_VERSION' >&2; exit 1; fi; actual=$(cat ${shellQuote(versionFile)}); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Database PostgreSQL major does not match requested version' >&2; exit 1; fi`, snapshotPaths(marker).mounts, + version, ); } yield* writeMarker({ ...marker, initialized: true }); }).pipe(Effect.mapError((cause) => errorFor("ready", cause))), ); - const removeData = Effect.fn("DockerDatabaseStorage.removeData")((_version: string) => + const removeData = Effect.fn("DockerDatabaseStorage.removeData")((version: string) => Effect.gen(function* () { const markerOption = yield* getMarkerForRemoval; if (Option.isNone(markerOption)) return; @@ -881,11 +916,15 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* runHelper( `set -eu; rm -rf /instance/data /instance/.supabase-restore-*; mkdir -p /instance/data; chown 100:101 /instance/data`, snapshotPaths(marker).mounts, + version, + true, ); } else { yield* runHelper( `set -eu; rm -rf ${shellQuote(`/store/${marker.namespace}/data`)}; mkdir -p ${shellQuote(`/store/${marker.namespace}/data`)}; chown 100:101 ${shellQuote(`/store/${marker.namespace}/data`)}`, [{ source: marker.volume ?? "", target: "/store", readOnly: false, type: "volume" }], + version, + true, ); } yield* options.fs @@ -896,7 +935,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* writeMarker({ ...marker, initialized: false }); }).pipe(Effect.mapError((cause) => errorFor("reset", cause))), ); - const destroyData = Effect.fn("DockerDatabaseStorage.destroyData")((_version: string) => + const destroyData = Effect.fn("DockerDatabaseStorage.destroyData")((version: string) => Effect.gen(function* () { const present = yield* getMarkerIfPresent; const markerOption = Option.isSome(present) ? present : yield* getMarkerForRemoval; @@ -906,6 +945,8 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* runHelper( `set -eu; rm -rf /instance/data /instance/.supabase-restore-* /instance/.supabase-database-ready.json`, snapshotPaths(marker).mounts, + version, + true, ); yield* removeHelper(); } else { @@ -919,9 +960,12 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ), ); if (volumeExists) - yield* runHelper(`set -eu; rm -rf ${shellQuote(`/store/${marker.namespace}`)}`, [ - { source: marker.volume, target: "/store", readOnly: false, type: "volume" }, - ]); + yield* runHelper( + `set -eu; rm -rf ${shellQuote(`/store/${marker.namespace}`)}`, + [{ source: marker.volume, target: "/store", readOnly: false, type: "volume" }], + version, + true, + ); yield* removeHelper(); } }).pipe(Effect.mapError((cause) => errorFor("destroy", cause))), @@ -1006,12 +1050,13 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") }); const cacheOwnership = store.backend === "host" - ? `; owner=$(stat -c "%u:%g" /cache); mkdir -p /cache/stack-database-snapshots-helper; chown "$owner" /cache/stack-database-snapshots-helper; chown -R "$owner" ${root}` + ? `; owner=$(/usr/bin/busybox stat -c "%u:%g" /cache); /usr/bin/busybox mkdir -p /cache/stack-database-snapshots-helper; /usr/bin/busybox chown "$owner" /cache/stack-database-snapshots-helper; /usr/bin/busybox chown -R "$owner" ${root}` : ""; - yield* runHelper( - `set -eu; mkdir -p ${root}; flock -x -w 120 ${shellQuote(`${root}/.lock`)} sh -eu -c ${shellQuote(`set -eu; mkdir -p ${root}/entries ${root}/stages; find ${root}/stages -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +; find ${root}/entries -mindepth 1 -maxdepth 1 -name '*.retired' -exec rm -rf -- {} +; trap 'rm -rf ${stage}${cacheOwnership}' EXIT; if [ -e ${source}/postmaster.pid ]; then echo 'Cannot save a running database snapshot' >&2; exit 1; fi; if [ ! -f ${source}/PG_VERSION ]; then echo 'Cannot save snapshot: PG_VERSION is missing' >&2; exit 1; fi; actual=$(cat ${source}/PG_VERSION); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Cannot save snapshot: PostgreSQL major does not match requested version' >&2; exit 1; fi; bad=$(find ${source} \\( ! -type f ! -type d \\) -print -quit); if [ -n "$bad" ]; then echo "Cannot save snapshot: unsupported filesystem entry $bad" >&2; exit 1; fi; rm -rf ${stage}; mkdir -p ${stage}; cp -a --reflink=auto ${source} ${stage}/data; printf '%s' ${shellQuote(descriptor)} > ${stage}/descriptor.json; if [ -e ${target} ]; then rm -rf ${target}.retired; mv ${target} ${target}.retired; if ! mv ${stage} ${target}; then mv ${target}.retired ${target}; exit 1; fi; else mv ${stage} ${target}; fi; rm -rf ${target}.retired; touch ${target}; find ${root}/entries -mindepth 1 -maxdepth 1 -type d ! -name ${digest} ! -name '*.retired' -printf '%T@ %p\\n' | sort -rn | tail -n +3 | cut -d' ' -f2- | xargs -r rm -rf`)}; rm -rf ${shellQuote(stage)}`, - paths.mounts, + const command = withSnapshotLock( + root, + `set -eu; /usr/bin/busybox mkdir -p ${shellQuote(`${root}/entries`)} ${shellQuote(`${root}/stages`)}; /usr/bin/busybox find ${shellQuote(`${root}/stages`)} -mindepth 1 -maxdepth 1 -exec /usr/bin/busybox rm -rf -- {} +; /usr/bin/busybox find ${shellQuote(`${root}/entries`)} -mindepth 1 -maxdepth 1 -name '*.retired' -exec /usr/bin/busybox rm -rf -- {} +; trap '/usr/bin/busybox rm -rf ${stage}${cacheOwnership}' EXIT; if [ -e ${shellQuote(`${source}/postmaster.pid`)} ]; then echo 'Cannot save a running database snapshot' >&2; exit 1; fi; if [ ! -f ${shellQuote(`${source}/PG_VERSION`)} ]; then echo 'Cannot save snapshot: PG_VERSION is missing' >&2; exit 1; fi; actual=$(/usr/bin/busybox cat ${shellQuote(`${source}/PG_VERSION`)}); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Cannot save snapshot: PostgreSQL major does not match requested version' >&2; exit 1; fi; bad=$(/usr/bin/busybox find ${shellQuote(source)} \\( ! -type f ! -type d \\) -print -quit); if [ -n "$bad" ]; then echo "Cannot save snapshot: unsupported filesystem entry $bad" >&2; exit 1; fi; /usr/bin/busybox rm -rf ${shellQuote(stage)}; /usr/bin/busybox mkdir -p ${shellQuote(stage)}; /usr/local/bin/cp -a --reflink=auto ${shellQuote(source)} ${shellQuote(`${stage}/data`)}; printf '%s' ${shellQuote(descriptor)} > ${shellQuote(`${stage}/descriptor.json`)}; if [ -e ${shellQuote(target)} ]; then /usr/bin/busybox rm -rf ${shellQuote(`${target}.retired`)}; /usr/bin/busybox mv ${shellQuote(target)} ${shellQuote(`${target}.retired`)}; if ! /usr/bin/busybox mv ${shellQuote(stage)} ${shellQuote(target)}; then /usr/bin/busybox mv ${shellQuote(`${target}.retired`)} ${shellQuote(target)}; exit 1; fi; else /usr/bin/busybox mv ${shellQuote(stage)} ${shellQuote(target)}; fi; /usr/bin/busybox rm -rf ${shellQuote(`${target}.retired`)}; /usr/bin/busybox touch ${shellQuote(target)}; /usr/bin/busybox find ${shellQuote(`${root}/entries`)} -mindepth 1 -maxdepth 1 -type d ! -name ${shellQuote(digest)} ! -name '*.retired' -exec /usr/bin/busybox stat -c '%y %n' {} + | /usr/bin/busybox sort -r | /usr/bin/busybox tail -n +3 | /usr/bin/busybox cut -d ' ' -f 4- | /usr/bin/busybox xargs -r /usr/bin/busybox rm -rf`, ); + yield* runHelper(command, paths.mounts, version); }).pipe(Effect.mapError((cause) => errorFor("snapshot", cause))), ); const restoreSnapshot = Effect.fn("DockerDatabaseStorage.restoreSnapshot")( @@ -1040,15 +1085,18 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") profile: "supabase", keyDigest: digest, }); - const targetSetup = store.initialized ? `test -d ${data}` : `mkdir -p ${data}`; + const targetSetup = store.initialized + ? `test -d ${shellQuote(data)}` + : `/usr/bin/busybox mkdir -p ${shellQuote(data)}`; const cacheOwnership = store.backend === "host" - ? `; owner=$(stat -c "%u:%g" /cache); mkdir -p /cache/stack-database-snapshots-helper; chown "$owner" /cache/stack-database-snapshots-helper; chown -R "$owner" ${root}` + ? `; owner=$(/usr/bin/busybox stat -c "%u:%g" /cache); /usr/bin/busybox mkdir -p /cache/stack-database-snapshots-helper; /usr/bin/busybox chown "$owner" /cache/stack-database-snapshots-helper; /usr/bin/busybox chown -R "$owner" ${root}` : ""; - const result = yield* runHelper( - `set -eu; mkdir -p ${root}; flock -x -w 120 ${shellQuote(`${root}/.lock`)} sh -eu -c ${shellQuote(`set -eu; mkdir -p ${root}/entries ${root}/stages; find ${root}/stages -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +; find ${root}/entries -mindepth 1 -maxdepth 1 -name '*.retired' -exec rm -rf -- {} +; trap 'rm -rf ${stage}${cacheOwnership}' EXIT; if ! ${targetSetup}; then echo 'Initialized restore target data directory is missing' >&2; exit 1; fi; bad=$(find ${data} -mindepth 1 -print -quit); if [ -n "$bad" ]; then echo NONEMPTY; exit 0; fi; if [ ! -d ${source} ]; then echo MISS; exit 0; fi; if [ ! -f ${source}/descriptor.json ]; then echo 'Snapshot descriptor is missing' >&2; exit 1; fi; actual=$(cat ${source}/descriptor.json); expected=${shellQuote(descriptor)}; if [ "$actual" != "$expected" ]; then echo 'Snapshot descriptor does not match requested identity' >&2; exit 1; fi; bad=$(find ${source}/data \\( ! -type f ! -type d \\) -print -quit); if [ -n "$bad" ]; then echo "Snapshot contains unsupported filesystem entry $bad" >&2; exit 1; fi; if [ -e ${source}/data/postmaster.pid ]; then echo 'Snapshot contains postmaster.pid' >&2; exit 1; fi; rm -rf ${stage}; cp -a --reflink=auto ${source}/data ${stage}; actual=$(cat ${stage}/PG_VERSION); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Snapshot PostgreSQL major does not match requested version' >&2; exit 1; fi; ${store.backend === "host" ? `chown -R 100:101 ${stage};` : ""} rmdir ${data}; mv ${stage} ${data}; touch ${source}; echo HIT`)}; rm -rf ${shellQuote(stage)}`, - paths.mounts, + const command = withSnapshotLock( + root, + `set -eu; /usr/bin/busybox mkdir -p ${shellQuote(`${root}/entries`)} ${shellQuote(`${root}/stages`)}; /usr/bin/busybox find ${shellQuote(`${root}/stages`)} -mindepth 1 -maxdepth 1 -exec /usr/bin/busybox rm -rf -- {} +; /usr/bin/busybox find ${shellQuote(`${root}/entries`)} -mindepth 1 -maxdepth 1 -name '*.retired' -exec /usr/bin/busybox rm -rf -- {} +; trap '/usr/bin/busybox rm -rf ${stage}${cacheOwnership}' EXIT; if ! ${targetSetup}; then echo 'Initialized restore target data directory is missing' >&2; exit 1; fi; bad=$(/usr/bin/busybox find ${shellQuote(data)} -mindepth 1 -print -quit); if [ -n "$bad" ]; then echo NONEMPTY; exit 0; fi; if [ ! -d ${shellQuote(source)} ]; then echo MISS; exit 0; fi; if [ ! -f ${shellQuote(`${source}/descriptor.json`)} ]; then echo 'Snapshot descriptor is missing' >&2; exit 1; fi; actual=$(/usr/bin/busybox cat ${shellQuote(`${source}/descriptor.json`)}); expected=${shellQuote(descriptor)}; if [ "$actual" != "$expected" ]; then echo 'Snapshot descriptor does not match requested identity' >&2; exit 1; fi; bad=$(/usr/bin/busybox find ${shellQuote(`${source}/data`)} \\( ! -type f ! -type d \\) -print -quit); if [ -n "$bad" ]; then echo "Snapshot contains unsupported filesystem entry $bad" >&2; exit 1; fi; if [ -e ${shellQuote(`${source}/data/postmaster.pid`)} ]; then echo 'Snapshot contains postmaster.pid' >&2; exit 1; fi; /usr/bin/busybox rm -rf ${shellQuote(stage)}; /usr/local/bin/cp -a --reflink=auto ${shellQuote(`${source}/data`)} ${shellQuote(stage)}; actual=$(/usr/bin/busybox cat ${shellQuote(`${stage}/PG_VERSION`)}); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Snapshot PostgreSQL major does not match requested version' >&2; exit 1; fi; ${store.backend === "host" ? `/usr/bin/busybox chown -R 100:101 ${shellQuote(stage)};` : ""} /usr/bin/busybox rmdir ${shellQuote(data)}; /usr/bin/busybox mv ${shellQuote(stage)} ${shellQuote(data)}; /usr/bin/busybox touch ${shellQuote(source)}; echo HIT`, ); + const result = yield* runHelper(command, paths.mounts, version); if (result === "MISS") return false; if (result === "NONEMPTY") return yield* errorFor("restore", "Restore target data directory must be empty");