From 9d7e55dfa1d901a87a4ff596ccbe8c7195152805 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Fri, 2 Oct 2026 20:07:29 +0200 Subject: [PATCH 001/142] ci(repo): run the test suite on stack-architecture branch pushes --- .github/workflows/branch-ci.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/branch-ci.yml diff --git a/.github/workflows/branch-ci.yml b/.github/workflows/branch-ci.yml new file mode 100644 index 0000000000..cacc708e15 --- /dev/null +++ b/.github/workflows/branch-ci.yml @@ -0,0 +1,25 @@ +name: branch-ci + +# Runs the Test suite on pushes to stack-architecture branches, which have no PR yet. +on: + push: + branches: + - stack-architecture + - stack-architecture/** + +permissions: + actions: read + contents: read + +concurrency: + group: branch-ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Test + uses: ./.github/workflows/test.yml + with: + force: true + secrets: + DF_FIREWALL_TOKEN: ${{ secrets.DF_FIREWALL_TOKEN }} From df2fe83b69c46ea6a4181b8869833ddb65862c17 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Fri, 2 Oct 2026 20:23:08 +0200 Subject: [PATCH 002/142] fix(cli): remove the functions deploy bundle directory on failure and interruption --- .../deploy/deploy.integration.test.ts | 134 ++++++++++- apps/cli/src/shared/functions/deploy.ts | 226 +++++++++--------- .../tests/helpers/child-process-spawner.ts | 9 +- 3 files changed, 252 insertions(+), 117 deletions(-) diff --git a/apps/cli/src/commands/functions/deploy/deploy.integration.test.ts b/apps/cli/src/commands/functions/deploy/deploy.integration.test.ts index 600907e826..358bfc1612 100644 --- a/apps/cli/src/commands/functions/deploy/deploy.integration.test.ts +++ b/apps/cli/src/commands/functions/deploy/deploy.integration.test.ts @@ -1,6 +1,20 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Effect, Exit, FileSystem, Layer, Option, Path, Stdio } from "effect"; +import { + Deferred, + Effect, + Exit, + Fiber, + FileSystem, + Layer, + Option, + Path, + Predicate, + Sink, + Stdio, + Stream, +} from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; import { YesFlag } from "../../../command-internal/global-flags.ts"; import { stripControlSequences } from "../../../shared/output/strip-control-sequences.ts"; @@ -1648,4 +1662,122 @@ describe("functions deploy", () => { }).pipe(Effect.provide(layer), Effect.ensuring(removeTempRoot)); }); }); + + describe("bundle temp directory cleanup", () => { + const readBundleOutputDirs = Effect.fnUntraced(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const entries = yield* fs.readDirectory(path.join(tempRoot.current, "supabase", ".temp")); + return entries.filter((entry) => entry.startsWith(".supabase-output-")); + }); + + /** Spawns `docker run` as a process that never exits until its fiber is interrupted. */ + function mockDockerRunHangingSpawner(onRunStarted: () => void) { + let nextPid = 0; + return Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => { + const cmd = Predicate.isTagged(command, "StandardCommand") ? command.command : ""; + const args = Predicate.isTagged(command, "StandardCommand") ? command.args : []; + const isRun = cmd === "docker" && args[0] === "run"; + nextPid += 1; + if (isRun) { + onRunStarted(); + } + return Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(9000 + nextPid), + stdout: Stream.empty, + stderr: Stream.empty, + all: Stream.empty, + exitCode: isRun ? Effect.never : Effect.succeed(ChildProcessSpawner.ExitCode(0)), + isRunning: Effect.succeed(true), + stdin: Sink.drain, + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }), + ); + }), + ); + } + + it.live("removes the bundle temp directory when the bundler container exits non-zero", () => { + const out = mockOutput({ format: "text" }); + const api = mockCommandPlatformApi({ + handler: (request) => Effect.succeed(jsonResponse(request, 200, [])), + }); + const child = mockChildProcessSpawner({ + exitCode: (record) => (record.command === "docker" && record.args[0] === "run" ? 1 : 0), + }); + const layer = Layer.mergeAll( + buildTestRuntime({ + out, + api, + cliSettings: mockCommandSettings({ workdir: tempRoot.current }), + runtimeInfo: mockRuntimeInfo({ cwd: tempRoot.current }), + }), + Layer.succeed(YesFlag, false), + child.layer, + Stdio.layerTest({ + args: Effect.succeed(["functions", "deploy", "hello-world", "--use-api=false"]), + }), + ); + + return Effect.gen(function* () { + yield* writeCliConfig(tempRoot.current); + yield* writeLocalFunction(tempRoot.current, "hello-world"); + + const error = yield* functionsDeploy({ + ...baseFlags, + useApi: false, + useDocker: true, + }).pipe(Effect.flip); + + expect(error).toBeInstanceOf(Error); + expect(error).toMatchObject({ message: "failed to bundle function: exit 1" }); + expect(yield* readBundleOutputDirs()).toEqual([]); + }).pipe(Effect.provide(layer), Effect.ensuring(removeTempRoot)); + }); + + it.live("removes the bundle temp directory when the deploy is interrupted mid-bundle", () => { + const out = mockOutput({ format: "text" }); + const api = mockCommandPlatformApi({ + handler: (request) => Effect.succeed(jsonResponse(request, 200, [])), + }); + const bundlerStarted = Deferred.makeUnsafe(); + const child = mockDockerRunHangingSpawner(() => { + Deferred.doneUnsafe(bundlerStarted, Exit.void); + }); + const layer = Layer.mergeAll( + buildTestRuntime({ + out, + api, + cliSettings: mockCommandSettings({ workdir: tempRoot.current }), + runtimeInfo: mockRuntimeInfo({ cwd: tempRoot.current }), + }), + Layer.succeed(YesFlag, false), + child, + Stdio.layerTest({ + args: Effect.succeed(["functions", "deploy", "hello-world", "--use-api=false"]), + }), + ); + + return Effect.gen(function* () { + yield* writeCliConfig(tempRoot.current); + yield* writeLocalFunction(tempRoot.current, "hello-world"); + + const fiber = yield* functionsDeploy({ + ...baseFlags, + useApi: false, + useDocker: true, + }).pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(bundlerStarted); + yield* Fiber.interrupt(fiber); + + expect(yield* readBundleOutputDirs()).toEqual([]); + }).pipe(Effect.provide(layer), Effect.ensuring(removeTempRoot)); + }); + }); }); diff --git a/apps/cli/src/shared/functions/deploy.ts b/apps/cli/src/shared/functions/deploy.ts index b86e82613d..ae2a57cbef 100644 --- a/apps/cli/src/shared/functions/deploy.ts +++ b/apps/cli/src/shared/functions/deploy.ts @@ -1443,121 +1443,125 @@ const bundleFunctionWithDocker = Effect.fn("functions.deploy.bundleWithDocker")( const outputRoot = resolve(functionsDir, "..", ".temp"); yield* Effect.tryPromise(() => mkdir(outputRoot, { recursive: true })); - const outputDir = yield* Effect.tryPromise(() => - mkdtemp(join(outputRoot, `.supabase-output-${config.slug}-`)), - ); - try { - // Go passes 0777 to MkdirAll, which Windows ignores. Calling chmod separately - // adds an NTFS WRITE_ATTRIBUTES requirement that the Go CLI does not have. - if (shouldChmodBundleOutputDirectory(process.platform)) { - yield* Effect.tryPromise({ - try: () => chmod(outputDir, 0o777), - catch: (cause) => (cause instanceof Error ? cause : new Error(String(cause))), - }); - } - const outputPath = join(outputDir, "output.eszip"); - // `edgeRuntimeImage` applies the tag verbatim — a `.temp/edge-runtime-version` pin flows - // through unmodified, `v` prefix or not (see the helper's doc in `functions.shared.ts`). - const rawImage = edgeRuntimeImage(edgeRuntimeVersion, yield* slimImagesEnabled); - const binds = yield* buildDockerBinds(projectId, functionsDir, outputDir, config, { - bitbucketCloneDirDefined, - onWarning: (message) => Effect.runPromise(output.raw(message, "stderr")), - }); - // Resolved per function rather than hoisted out of the loop (unlike `download.ts`'s - // `PulledEdgeRuntimeImage`): the first resolve failure aborts the loop, and the only added - // cost is one cached `docker image inspect` per function. - const image = yield* resolveFunctionsDockerImage(rawImage, projectEnvValues); - yield* ensureDockerNetwork(networkMode, projectId); - yield* ensureDockerNamedVolume( - edgeRuntimeCacheVolume(projectId).name, - projectId, - projectEnvValues, - ); - - const env: Array = []; - if ( - !(yield* Effect.promise(() => - shouldUsePackageJsonDiscovery(config.entrypoint, config.importMap), - )) - ) { - env.push("DENO_NO_PACKAGE_JSON=1"); - } - env.push(...dockerNpmEnv()); + // A generator `finally` does not run when a yielded effect fails or is interrupted. + return yield* Effect.acquireUseRelease( + Effect.tryPromise(() => mkdtemp(join(outputRoot, `.supabase-output-${config.slug}-`))), + (outputDir) => + Effect.gen(function* () { + // Go passes 0777 to MkdirAll, which Windows ignores. Calling chmod separately + // adds an NTFS WRITE_ATTRIBUTES requirement that the Go CLI does not have. + if (shouldChmodBundleOutputDirectory(process.platform)) { + yield* Effect.tryPromise({ + try: () => chmod(outputDir, 0o777), + catch: (cause) => (cause instanceof Error ? cause : new Error(String(cause))), + }); + } + const outputPath = join(outputDir, "output.eszip"); + // `edgeRuntimeImage` applies the tag verbatim — a `.temp/edge-runtime-version` pin flows + // through unmodified, `v` prefix or not (see the helper's doc in `functions.shared.ts`). + const rawImage = edgeRuntimeImage(edgeRuntimeVersion, yield* slimImagesEnabled); + const binds = yield* buildDockerBinds(projectId, functionsDir, outputDir, config, { + bitbucketCloneDirDefined, + onWarning: (message) => Effect.runPromise(output.raw(message, "stderr")), + }); + // Resolved per function rather than hoisted out of the loop (unlike `download.ts`'s + // `PulledEdgeRuntimeImage`): the first resolve failure aborts the loop, and the only added + // cost is one cached `docker image inspect` per function. + const image = yield* resolveFunctionsDockerImage(rawImage, projectEnvValues); + yield* ensureDockerNetwork(networkMode, projectId); + yield* ensureDockerNamedVolume( + edgeRuntimeCacheVolume(projectId).name, + projectId, + projectEnvValues, + ); - const containerArgs = [ - "bundle", - "--entrypoint", - toDockerPath(config.entrypoint, { resolve }), - "--output", - toDockerPath(outputPath, { resolve }), - ]; - if ( - config.importMap.length > 0 && - !shouldUseDenoJsonDiscovery(config.entrypoint, config.importMap) - ) { - containerArgs.push("--import-map", toDockerPath(config.importMap, { resolve })); - } - for (const staticFile of config.staticFiles) { - containerArgs.push("--static", toDockerPath(staticFile, { resolve })); - } - if (verbose || process.env["DEBUG"] === "true") { - containerArgs.push("--verbose"); - } + const env: Array = []; + if ( + !(yield* Effect.promise(() => + shouldUsePackageJsonDiscovery(config.entrypoint, config.importMap), + )) + ) { + env.push("DENO_NO_PACKAGE_JSON=1"); + } + env.push(...dockerNpmEnv()); + + const containerArgs = [ + "bundle", + "--entrypoint", + toDockerPath(config.entrypoint, { resolve }), + "--output", + toDockerPath(outputPath, { resolve }), + ]; + if ( + config.importMap.length > 0 && + !shouldUseDenoJsonDiscovery(config.entrypoint, config.importMap) + ) { + containerArgs.push("--import-map", toDockerPath(config.importMap, { resolve })); + } + for (const staticFile of config.staticFiles) { + containerArgs.push("--static", toDockerPath(staticFile, { resolve })); + } + if (verbose || process.env["DEBUG"] === "true") { + containerArgs.push("--verbose"); + } - const command = buildFunctionsDockerRunArgs({ - image, - projectId, - networkMode, - binds: binds.map(formatDockerBind), - env, - // `functionsDir` is `/supabase/functions`, same derivation as `deployViaApi`'s - // own `projectRoot`. - workingDir: toDockerPath(resolve(functionsDir, "..", ".."), { resolve }), - containerArgs, - }); + const command = buildFunctionsDockerRunArgs({ + image, + projectId, + networkMode, + binds: binds.map(formatDockerBind), + env, + // `functionsDir` is `/supabase/functions`, same derivation as `deployViaApi`'s + // own `projectRoot`. + workingDir: toDockerPath(resolve(functionsDir, "..", ".."), { resolve }), + containerArgs, + }); - // Live-tees each chunk to `output.raw` as it arrives, rather than buffering the whole run - // until exit. - const result = yield* runChildProcess("docker", command, { - stdout: "pipe", - stderr: "pipe", - onStdout: (chunk) => output.raw(chunk, output.format === "text" ? "stdout" : "stderr"), - onStderr: (chunk) => output.raw(chunk, "stderr"), - }); - if (result.exitCode !== 0) { - return yield* Effect.fail(new Error(`failed to bundle function: exit ${result.exitCode}`)); - } + // Live-tees each chunk to `output.raw` as it arrives, rather than buffering the whole run + // until exit. + const result = yield* runChildProcess("docker", command, { + stdout: "pipe", + stderr: "pipe", + onStdout: (chunk) => output.raw(chunk, output.format === "text" ? "stdout" : "stderr"), + onStderr: (chunk) => output.raw(chunk, "stderr"), + }); + if (result.exitCode !== 0) { + return yield* Effect.fail( + new Error(`failed to bundle function: exit ${result.exitCode}`), + ); + } - const eszip = yield* Effect.tryPromise({ - try: () => readFile(outputPath), - catch: (error) => - new Error( - `failed to open eszip: ${error instanceof Error ? error.message : String(error)}`, - ), - }); - const compressed = new Uint8Array( - Buffer.concat([ - Buffer.from(COMPRESSED_ESZIP_MAGIC), - brotliCompressSync(eszip, { - params: { - [zlibConstants.BROTLI_PARAM_QUALITY]: 6, - }, - }), - ]), - ); - const sha256 = yield* Effect.promise(() => crypto.subtle.digest("SHA-256", compressed)); - const hash = Buffer.from(sha256).toString("hex"); - yield* Effect.annotateCurrentSpan({ "bundle.bytes": compressed.byteLength }); - return { - slug: config.slug, - metadata: createBundledMetadata(config, hash), - body: compressed, - } satisfies BundledFunction; - } finally { - yield* Effect.tryPromise(() => rm(outputDir, { recursive: true, force: true })).pipe( - Effect.orElseSucceed(() => undefined), - ); - } + const eszip = yield* Effect.tryPromise({ + try: () => readFile(outputPath), + catch: (error) => + new Error( + `failed to open eszip: ${error instanceof Error ? error.message : String(error)}`, + ), + }); + const compressed = new Uint8Array( + Buffer.concat([ + Buffer.from(COMPRESSED_ESZIP_MAGIC), + brotliCompressSync(eszip, { + params: { + [zlibConstants.BROTLI_PARAM_QUALITY]: 6, + }, + }), + ]), + ); + const sha256 = yield* Effect.promise(() => crypto.subtle.digest("SHA-256", compressed)); + const hash = Buffer.from(sha256).toString("hex"); + yield* Effect.annotateCurrentSpan({ "bundle.bytes": compressed.byteLength }); + return { + slug: config.slug, + metadata: createBundledMetadata(config, hash), + body: compressed, + } satisfies BundledFunction; + }), + (outputDir) => + Effect.tryPromise(() => rm(outputDir, { recursive: true, force: true })).pipe( + Effect.orElseSucceed(() => undefined), + ), + ); }); const listRemoteFunctions = Effect.fn("functions.deploy.listRemoteFunctions")(function* ( diff --git a/apps/cli/tests/helpers/child-process-spawner.ts b/apps/cli/tests/helpers/child-process-spawner.ts index 7dc2ad5a0e..93200bae25 100644 --- a/apps/cli/tests/helpers/child-process-spawner.ts +++ b/apps/cli/tests/helpers/child-process-spawner.ts @@ -29,7 +29,7 @@ const isOneShotSupervisor = (args: ReadonlyArray): boolean => { export function mockChildProcessSpawner( opts: { - exitCode?: number; + exitCode?: number | ((record: SpawnRecord) => number); stdout?: string[]; stderr?: string[]; beforeSpawn?: (record: SpawnRecord) => Effect.Effect; @@ -63,10 +63,9 @@ export function mockChildProcessSpawner( cmd === process.execPath && !isOneShotSupervisor(args) ? "30 seconds" : "10 millis", ); running = false; - yield* Deferred.succeed( - exitDeferred, - ChildProcessSpawner.ExitCode(opts.exitCode ?? 0), - ); + const resolvedExitCode = + typeof opts.exitCode === "function" ? opts.exitCode(record) : (opts.exitCode ?? 0); + yield* Deferred.succeed(exitDeferred, ChildProcessSpawner.ExitCode(resolvedExitCode)); }), ); From ec079c9b3e750ae2b97b2713309ace1e08be01f5 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Fri, 2 Oct 2026 20:26:20 +0200 Subject: [PATCH 003/142] ci(stack): add native platform scenarios and a repeated idle/parallel stress gate --- .github/workflows/test.yml | 85 +++++++++++++++++++++++++++++++++++-- packages/stack/package.json | 1 + 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3c934ecae5..cb0fdf54f1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -174,6 +174,39 @@ jobs: src/commands/experimental/stack/start/start.integration.test.ts --passWithNoTests=false + test-stack-native-scenarios: + if: | + !startsWith(github.head_ref, 'release-notes/') && + (github.event_name == 'merge_group' || + inputs.force || + github.event.pull_request.draft == false) + name: Run stack native scenarios (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + # GitHub-hosted macOS runners have no Docker and Windows runners can't + # run Linux containers, so only the native backend's whole-stack e2e + # entry files run here. Linux coverage for both backends comes from + # test-stack-e2e. + os: [macos-latest, windows-latest] + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + go-cache: "false" + + - name: Run stack native whole-stack scenarios + working-directory: packages/stack + run: pnpm test:e2e:native-scenarios --passWithNoTests=false + test-summary: if: | always() && @@ -305,6 +338,47 @@ jobs: src/public.e2e.test.ts --passWithNoTests=false + # Repeats the idle/wake and parallel-stack scenarios back to back on both + # backends so timing-dependent races surface before they reach users. A + # failing iteration fails the job outright; it is never retried. + test-stack-stress: + if: | + !startsWith(github.head_ref, 'release-notes/') && + (github.event_name == 'merge_group' || + inputs.force || + github.event.pull_request.draft == false) + name: Run stack stress scenarios + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 90 + env: + # Runs per backend; vitest's --repeats counts the runs after the first. + STACK_STRESS_ITERATIONS: 10 + steps: + - name: Checkout + uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1.0.0-beta + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + go-cache: "false" + + - name: Run native idle and parallel-stack scenarios repeatedly + working-directory: packages/stack + run: >- + pnpm run test:e2e:run + whole-stack.native.idle-parallel + --repeats=$((STACK_STRESS_ITERATIONS - 1)) + --passWithNoTests=false + + - name: Run Docker idle and parallel-stack scenarios repeatedly + working-directory: packages/stack + run: >- + pnpm run test:e2e:run + whole-stack.docker.idle-parallel + --repeats=$((STACK_STRESS_ITERATIONS - 1)) + --passWithNoTests=false + # Summary job that gates branch protection. It preserves the original check # name and succeeds iff every CLI shard and stack runtime succeeded. test-e2e-summary: @@ -315,13 +389,16 @@ jobs: inputs.force || github.event.pull_request.draft == false) name: Run end-to-end tests - needs: [test-e2e, test-stack-e2e] + needs: [test-e2e, test-stack-e2e, test-stack-native-scenarios, test-stack-stress] runs-on: ubuntu-latest steps: - name: Verify all shards succeeded run: | - if [ "${{ needs.test-e2e.result }}" = "failure" ] || [ "${{ needs.test-e2e.result }}" = "cancelled" ] || [ "${{ needs.test-stack-e2e.result }}" = "failure" ] || [ "${{ needs.test-stack-e2e.result }}" = "cancelled" ]; then - echo "::error ::One or more e2e jobs failed: cli=${{ needs.test-e2e.result }}, stack=${{ needs.test-stack-e2e.result }}" + if [ "${{ needs.test-e2e.result }}" = "failure" ] || [ "${{ needs.test-e2e.result }}" = "cancelled" ] || \ + [ "${{ needs.test-stack-e2e.result }}" = "failure" ] || [ "${{ needs.test-stack-e2e.result }}" = "cancelled" ] || \ + [ "${{ needs.test-stack-native-scenarios.result }}" = "failure" ] || [ "${{ needs.test-stack-native-scenarios.result }}" = "cancelled" ] || \ + [ "${{ needs.test-stack-stress.result }}" = "failure" ] || [ "${{ needs.test-stack-stress.result }}" = "cancelled" ]; then + echo "::error ::One or more e2e jobs failed: cli=${{ needs.test-e2e.result }}, stack=${{ needs.test-stack-e2e.result }}, stack-native-scenarios=${{ needs.test-stack-native-scenarios.result }}, stack-stress=${{ needs.test-stack-stress.result }}" exit 1 fi - echo "All e2e jobs reported: cli=${{ needs.test-e2e.result }}, stack=${{ needs.test-stack-e2e.result }}" + echo "All e2e jobs reported: cli=${{ needs.test-e2e.result }}, stack=${{ needs.test-stack-e2e.result }}, stack-native-scenarios=${{ needs.test-stack-native-scenarios.result }}, stack-stress=${{ needs.test-stack-stress.result }}" diff --git a/packages/stack/package.json b/packages/stack/package.json index e71911609f..69afe969c5 100644 --- a/packages/stack/package.json +++ b/packages/stack/package.json @@ -23,6 +23,7 @@ "test:integration:run": "bun --bun vitest run --project integration", "test:e2e": "pnpm run test:e2e:run", "test:e2e:run": "bun --bun vitest run --project e2e", + "test:e2e:native-scenarios": "pnpm run test:e2e:run whole-stack.native", "types:check": "tsc --noEmit" }, "dependencies": { From 9acb1debeafcd1ddae207638faac4369f48aa41c Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Fri, 2 Oct 2026 20:52:16 +0200 Subject: [PATCH 004/142] ci(stack): run native platform scenarios on macOS only --- .github/workflows/test.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cb0fdf54f1..0caa0635d6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -186,11 +186,9 @@ jobs: strategy: fail-fast: false matrix: - # GitHub-hosted macOS runners have no Docker and Windows runners can't - # run Linux containers, so only the native backend's whole-stack e2e - # entry files run here. Linux coverage for both backends comes from - # test-stack-e2e. - os: [macos-latest, windows-latest] + # Native artifacts exist for macOS and Linux only, and macOS runners have + # no Docker; Linux coverage for both backends comes from test-stack-e2e. + os: [macos-latest] steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 From ce745bbf2dd472eeb4471bd266d18ea7f6b61547 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Fri, 2 Oct 2026 22:44:59 +0200 Subject: [PATCH 005/142] refactor(cli): make functions deploy idiomatic Effect with typed errors and schedules --- .oxlintrc.effect.json | 1 + .../deploy/deploy.integration.test.ts | 441 ++++- .../cli/src/shared/functions/deploy.errors.ts | 118 +- apps/cli/src/shared/functions/deploy.ts | 1508 +++++++++-------- .../src/shared/functions/deploy.unit.test.ts | 62 +- apps/cli/src/shared/functions/serve.ts | 4 +- 6 files changed, 1367 insertions(+), 767 deletions(-) diff --git a/.oxlintrc.effect.json b/.oxlintrc.effect.json index 137f3fb227..2f9ab05d35 100644 --- a/.oxlintrc.effect.json +++ b/.oxlintrc.effect.json @@ -12,6 +12,7 @@ "!apps/cli/src/shared/compute/**", "!apps/cli/src/shared/config/**", "!apps/cli/src/shared/feedback/**", + "!apps/cli/src/shared/functions/deploy.ts", "!apps/cli/src/shared/functions/functions-docker.ts", "!apps/cli/src/shared/functions/functions-docker.unit.test.ts", "!apps/cli/src/shared/functions/serve.ts", diff --git a/apps/cli/src/commands/functions/deploy/deploy.integration.test.ts b/apps/cli/src/commands/functions/deploy/deploy.integration.test.ts index 358bfc1612..20358f03f3 100644 --- a/apps/cli/src/commands/functions/deploy/deploy.integration.test.ts +++ b/apps/cli/src/commands/functions/deploy/deploy.integration.test.ts @@ -1,7 +1,9 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { + Clock, Deferred, + Duration, Effect, Exit, Fiber, @@ -10,11 +12,16 @@ import { Option, Path, Predicate, + Queue, + Ref, + Schema, Sink, Stdio, Stream, } from "effect"; import { ChildProcessSpawner } from "effect/unstable/process"; +import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; +import * as TestClock from "effect/testing/TestClock"; import { YesFlag } from "../../../command-internal/global-flags.ts"; import { stripControlSequences } from "../../../shared/output/strip-control-sequences.ts"; @@ -35,9 +42,11 @@ import { shouldChmodBundleOutputDirectory, } from "../../../shared/functions/deploy.ts"; import { toDockerPath } from "../../../shared/functions/functions-docker.ts"; +import { FunctionsApiStatusError } from "../../../shared/functions/functions-api.errors.ts"; import { functionsGoConfigCompat } from "../../../command-internal/functions-go-config.ts"; import { ConflictingFunctionDeployFlagsError, + FunctionAssetOutsideRootError, InvalidFunctionDeploySlugError, NoFunctionsToDeployError, } from "../../../shared/functions/deploy.errors.ts"; @@ -47,6 +56,7 @@ import { functionsDeploy } from "./deploy.handler.ts"; import type { FunctionsDeployFlags } from "./deploy.command.ts"; const tempRoot = useTempWorkdir("supabase-functions-deploy-legacy-"); +const isRoot = typeof process.getuid === "function" && process.getuid() === 0; const baseFlags: FunctionsDeployFlags = { functionNames: ["hello-world"], @@ -837,7 +847,7 @@ describe("functions deploy", () => { const error = yield* functionsDeploy({ ...baseFlags, useDocker: true }).pipe(Effect.flip); expect(error).toBeInstanceOf(ConflictingFunctionDeployFlagsError); - if (!(error instanceof ConflictingFunctionDeployFlagsError)) { + if (!Schema.is(ConflictingFunctionDeployFlagsError)(error)) { throw new Error(`unexpected error: ${String(error)}`); } expect(error.message).toBe( @@ -1145,7 +1155,7 @@ describe("functions deploy", () => { const error = yield* deployNoFunctions.pipe(Effect.flip); expect(error).toBeInstanceOf(NoFunctionsToDeployError); - if (!(error instanceof NoFunctionsToDeployError)) { + if (!Schema.is(NoFunctionsToDeployError)(error)) { throw new Error(`unexpected error: ${String(error)}`); } expect(error.message).toBe( @@ -1780,4 +1790,431 @@ describe("functions deploy", () => { }).pipe(Effect.provide(layer), Effect.ensuring(removeTempRoot)); }); }); + + it.live( + "does not retry a create response whose success status decodes to a malformed body", + () => { + const out = mockOutput({ format: "text" }); + let createCalls = 0; + const api = mockCommandPlatformApi({ + handler: (request) => { + if (request.method === "GET") { + return Effect.succeed(jsonResponse(request, 200, [])); + } + createCalls += 1; + return Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response("not valid json", { + status: 201, + headers: { "content-type": "application/json" }, + }), + ), + ); + }, + }); + const child = mockDockerBundleSpawner(); + const layer = Layer.mergeAll( + buildTestRuntime({ + out, + api, + cliSettings: mockCommandSettings({ workdir: tempRoot.current }), + runtimeInfo: mockRuntimeInfo({ cwd: tempRoot.current }), + }), + Layer.succeed(YesFlag, false), + child.layer, + Stdio.layerTest({ + args: Effect.succeed(["functions", "deploy", "hello-world", "--use-api=false"]), + }), + ); + + return Effect.gen(function* () { + yield* writeCliConfig(tempRoot.current); + yield* writeLocalFunction(tempRoot.current, "hello-world"); + + const error = yield* functionsDeploy({ + ...baseFlags, + useApi: false, + useDocker: true, + }).pipe(Effect.flip); + + expect(error).toBeInstanceOf(FunctionsApiStatusError); + if (!(error instanceof FunctionsApiStatusError)) { + throw new Error(`unexpected error: ${String(error)}`); + } + expect(error.message).toContain("failed to read function response:"); + // Exactly one create request: a malformed success body is terminal, not retried. + expect(createCalls).toBe(1); + }).pipe(Effect.provide(layer), Effect.ensuring(removeTempRoot)); + }, + ); + + describe("transient API retries (Schedule-based backoff)", () => { + // `it.effect` (not `it.live`): the local filesystem setup below runs to completion with real + // I/O before `functionsDeploy` is forked, so nothing timed races against real I/O once + // `TestClock` takes over. The underlying `@supabase/api` client retries a transient response + // immediately (no delay — see `packages/api/src/internal/client.ts`'s `applySupabaseRetryPolicy`, + // `maxRetries` defaulting to 5, i.e. 6 raw requests per logical call), so each of + // `listRemoteFunctions`'s own 4 attempts produces 6 requests; only its own 1s/2s/4s + // `Schedule.exponential` backoff between attempts actually sleeps. + it.effect( + "gives up listing remote functions after exhausting the retry bound on persistent 5xx, " + + "honoring the exponential backoff and the API client's own transient retry", + () => + Effect.gen(function* () { + const out = mockOutput({ format: "text" }); + const requestCount = yield* Ref.make(0); + // Offered by the fake handler itself on every raw GET, so the test can wait on an + // observable signal instead of polling real time or guessing how many scheduler ticks + // a response takes to decode. + const requestSeen = yield* Queue.unbounded(); + const api = mockCommandPlatformApi({ + handler: (request) => { + if (request.method === "GET") { + return Ref.update(requestCount, (n) => n + 1).pipe( + Effect.andThen(Queue.offer(requestSeen, undefined)), + Effect.as(jsonResponse(request, 503, { message: "unavailable" })), + ); + } + return Effect.succeed(jsonResponse(request, 201, {})); + }, + }); + const layer = Layer.mergeAll( + buildTestRuntime({ + out, + api, + cliSettings: mockCommandSettings({ workdir: tempRoot.current }), + runtimeInfo: mockRuntimeInfo({ cwd: tempRoot.current }), + }), + Layer.succeed(YesFlag, false), + Stdio.layerTest({ + args: Effect.succeed(["functions", "deploy", "hello-world", "--use-api"]), + }), + ); + + yield* writeCliConfig(tempRoot.current).pipe(Effect.provide(layer)); + yield* writeLocalFunction(tempRoot.current, "hello-world").pipe(Effect.provide(layer)); + + // `TestClock.adjust` doesn't wait for the response decode, so advance only after the + // backoff sleep is registered: the immediate fork records it before it can suspend. + const sleepRegistered = yield* Queue.unbounded(); + const realClock = yield* Effect.clockWith(Effect.succeed); + const instrumentedClock: Clock.Clock = { + ...realClock, + sleep: (duration) => + Effect.gen(function* () { + const sleeping = yield* realClock + .sleep(duration) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Queue.offer(sleepRegistered, undefined); + yield* Fiber.join(sleeping); + }), + }; + + const fiber = yield* functionsDeploy(baseFlags).pipe( + Effect.flip, + Effect.provide(layer), + Effect.provideService(Clock.Clock, instrumentedClock), + Effect.forkChild({ startImmediately: true }), + ); + + // Waits for the API client's own 6-request transient retry to fully land before + // trusting `requestCount`: each `Queue.take` suspends until the handler's offer for + // that raw request has actually been processed, so this also drains the real, unmocked + // filesystem reads the config/discovery steps ahead of the first attempt perform — + // no real-timer polling or `TestClock` involved yet. + const awaitRequestBatch = Effect.all( + Array.from({ length: 6 }, () => Queue.take(requestSeen)), + { discard: true }, + ); + + yield* awaitRequestBatch; + expect(yield* Ref.get(requestCount)).toBe(6); + + // Each subsequent attempt is gated by `listRemoteFunctions`'s own exponential delay. + // `TestClock.adjust` only runs fibers whose wake time has been reached: stopping 1ms + // short of each deadline and asserting the count is still unchanged tells 1s/2s/4s + // backoff apart from any other interval (e.g. 0.5s/1s/2s would have already fired). + let expectedCount = 6; + for (const delayMs of [1_000, 2_000, 4_000]) { + yield* Queue.take(sleepRegistered); + yield* TestClock.adjust(Duration.millis(delayMs - 1)); + expect(yield* Ref.get(requestCount)).toBe(expectedCount); + yield* TestClock.adjust(Duration.millis(1)); + yield* awaitRequestBatch; + expectedCount += 6; + expect(yield* Ref.get(requestCount)).toBe(expectedCount); + } + expect(expectedCount).toBe(24); + + const error = yield* Fiber.join(fiber); + if (!(error instanceof FunctionsApiStatusError)) { + throw new Error(`unexpected error: ${String(error)}`); + } + expect(error.message).toBe( + 'unexpected list functions status 503: {"message":"unavailable"}', + ); + }).pipe(Effect.ensuring(removeTempRoot)), + ); + }); + + it.live("rejects a static file outside the source root with the existing message", () => { + const out = mockOutput({ format: "text" }); + const api = mockCommandPlatformApi({ + handler: (request) => Effect.succeed(jsonResponse(request, 200, [])), + }); + const layer = Layer.mergeAll( + buildTestRuntime({ + out, + api, + cliSettings: mockCommandSettings({ workdir: tempRoot.current }), + runtimeInfo: mockRuntimeInfo({ cwd: tempRoot.current }), + }), + Layer.succeed(YesFlag, false), + Stdio.layerTest({ + args: Effect.succeed(["functions", "deploy", "hello-world", "--use-api"]), + }), + ); + + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const outsideDir = yield* fs.makeTempDirectoryScoped({ prefix: "deploy-outside-" }); + const outsideFile = path.join(outsideDir, "secret.txt"); + yield* fs.writeFileString(outsideFile, "do not upload\n"); + + yield* writeCliConfig( + tempRoot.current, + `project_id = "test-project"\n\n[functions.hello-world]\nstatic_files = ["${outsideFile}"]\n`, + ); + yield* writeLocalFunction(tempRoot.current, "hello-world"); + + const error = yield* functionsDeploy(baseFlags).pipe(Effect.flip); + + expect(Schema.is(FunctionAssetOutsideRootError)(error)).toBe(true); + if (!Schema.is(FunctionAssetOutsideRootError)(error)) { + throw new Error(`unexpected error: ${String(error)}`); + } + expect(error.message).toBe(`refusing to upload asset outside source root: ${outsideFile}`); + }).pipe(Effect.provide(layer), Effect.scoped, Effect.ensuring(removeTempRoot)); + }); + + const functionDeployResponseBody = { + id: "function-id", + slug: "hello-world", + name: "hello-world", + status: "ACTIVE", + version: 2, + created_at: 1_687_423_025_152, + updated_at: 1_687_423_025_152, + verify_jwt: true, + import_map: true, + entrypoint_path: "functions/hello-world/index.ts", + import_map_path: "functions/hello-world/deno.json", + }; + + // A chmod 000 directory is a no-op under root, which would make the glob resolve normally + // instead of exercising the failure this test targets. + it.live.skipIf(isRoot)( + "warns with the real filesystem error and continues past an unreadable nested directory under a static_files glob", + () => { + const out = mockOutput({ format: "text" }); + const api = mockCommandPlatformApi({ + handler: (request) => { + if (request.method === "GET") { + return Effect.succeed(jsonResponse(request, 200, [])); + } + return Effect.succeed(jsonResponse(request, 201, functionDeployResponseBody)); + }, + }); + const layer = Layer.mergeAll( + buildTestRuntime({ + out, + api, + cliSettings: mockCommandSettings({ workdir: tempRoot.current }), + runtimeInfo: mockRuntimeInfo({ cwd: tempRoot.current }), + }), + Layer.succeed(YesFlag, false), + Stdio.layerTest({ + args: Effect.succeed(["functions", "deploy", "hello-world", "--use-api"]), + }), + ); + + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const staticRoot = path.join(tempRoot.current, "supabase", "static"); + const lockedDir = path.join(staticRoot, "locked"); + yield* fs.makeDirectory(lockedDir, { recursive: true }); + yield* fs.writeFileString(path.join(staticRoot, "visible.txt"), "visible\n"); + yield* fs.writeFileString(path.join(lockedDir, "secret.txt"), "secret\n"); + // Only the nested directory is unreadable — a reader can still list `static/` itself, so + // a bypassed chmod (not expected here, guarded by `skipIf(isRoot)`) would surface as an + // actual upload of both files rather than a silent pass. + yield* Effect.acquireRelease(fs.chmod(lockedDir, 0o000), () => + fs.chmod(lockedDir, 0o700).pipe(Effect.orDie), + ); + + yield* writeCliConfig( + tempRoot.current, + 'project_id = "test-project"\n\n[functions.hello-world]\nstatic_files = ["static/**/*.txt"]\n', + ); + yield* writeLocalFunction(tempRoot.current, "hello-world"); + + yield* functionsDeploy(baseFlags); + + expect(out.stderrText).toContain(`WARN: EACCES: permission denied, open '${lockedDir}'`); + expect(stripControlSequences(out.stdoutText)).toContain("Deployed Functions on project"); + }).pipe(Effect.provide(layer), Effect.scoped, Effect.ensuring(removeTempRoot)); + }, + ); + + it.live( + "warns and continues past a static_files glob that is not a valid regular expression", + () => { + const out = mockOutput({ format: "text" }); + const api = mockCommandPlatformApi({ + handler: (request) => { + if (request.method === "GET") { + return Effect.succeed(jsonResponse(request, 200, [])); + } + return Effect.succeed(jsonResponse(request, 201, functionDeployResponseBody)); + }, + }); + const layer = Layer.mergeAll( + buildTestRuntime({ + out, + api, + cliSettings: mockCommandSettings({ workdir: tempRoot.current }), + runtimeInfo: mockRuntimeInfo({ cwd: tempRoot.current }), + }), + Layer.succeed(YesFlag, false), + Stdio.layerTest({ + args: Effect.succeed(["functions", "deploy", "hello-world", "--use-api"]), + }), + ); + + return Effect.gen(function* () { + // `[z-a]` is a glob bracket expression with an out-of-order range: valid Deno/Go glob + // syntax, but `new RegExp(...)` rejects it as an invalid character class. + yield* writeCliConfig( + tempRoot.current, + 'project_id = "test-project"\n\n[functions.hello-world]\nstatic_files = ["[z-a].txt"]\n', + ); + yield* writeLocalFunction(tempRoot.current, "hello-world"); + + yield* functionsDeploy(baseFlags); + + expect(out.stderrText).toContain("WARN:"); + expect(stripControlSequences(out.stdoutText)).toContain("Deployed Functions on project"); + }).pipe(Effect.provide(layer), Effect.ensuring(removeTempRoot)); + }, + ); + + it.live("deploys a function whose import map is a commented deno.jsonc", () => { + const out = mockOutput({ format: "text" }); + const api = mockCommandPlatformApi({ + handler: (request) => { + if (request.method === "GET") { + return Effect.succeed(jsonResponse(request, 200, [])); + } + return Effect.succeed( + jsonResponse(request, 201, { + id: "function-id", + slug: "hello-world", + name: "hello-world", + status: "ACTIVE", + version: 2, + created_at: 1_687_423_025_152, + updated_at: 1_687_423_025_152, + verify_jwt: true, + import_map: true, + entrypoint_path: "functions/hello-world/index.ts", + import_map_path: "functions/hello-world/deno.jsonc", + }), + ); + }, + }); + const layer = Layer.mergeAll( + buildTestRuntime({ + out, + api, + cliSettings: mockCommandSettings({ workdir: tempRoot.current }), + runtimeInfo: mockRuntimeInfo({ cwd: tempRoot.current }), + }), + Layer.succeed(YesFlag, false), + Stdio.layerTest({ + args: Effect.succeed(["functions", "deploy", "hello-world", "--use-api"]), + }), + ); + + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* writeCliConfig(tempRoot.current); + const functionDir = path.join(tempRoot.current, "supabase", "functions", "hello-world"); + yield* fs.makeDirectory(functionDir, { recursive: true }); + yield* fs.writeFileString( + path.join(functionDir, "index.ts"), + "Deno.serve(() => new Response())\n", + ); + // Only a `.jsonc` import map exists (no `deno.json`), so discovery must fall through to it. + yield* fs.writeFileString( + path.join(functionDir, "deno.jsonc"), + ["// top-level comment", "{", " // nested comment", ' "imports": {}', "}", ""].join("\n"), + ); + + yield* functionsDeploy(baseFlags); + + expect(stripControlSequences(out.stdoutText)).toContain("Deployed Functions on project"); + }).pipe(Effect.provide(layer), Effect.ensuring(removeTempRoot)); + }); + + it.live( + "fails with the pre-refactor raw-errno message for a dangling symlink inside an import-map scope target", + () => { + const out = mockOutput({ format: "text" }); + const api = mockCommandPlatformApi({ + handler: (request) => Effect.succeed(jsonResponse(request, 200, [])), + }); + const layer = Layer.mergeAll( + buildTestRuntime({ + out, + api, + cliSettings: mockCommandSettings({ workdir: tempRoot.current }), + runtimeInfo: mockRuntimeInfo({ cwd: tempRoot.current }), + }), + Layer.succeed(YesFlag, false), + Stdio.layerTest({ + args: Effect.succeed(["functions", "deploy", "hello-world", "--use-api"]), + }), + ); + + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* writeLocalFunction(tempRoot.current, "hello-world"); + yield* writeCliConfig(tempRoot.current); + const functionDir = path.join(tempRoot.current, "supabase", "functions", "hello-world"); + yield* fs.writeFileString( + path.join(functionDir, "deno.json"), + '{"imports":{},"scopes":{"x/":{"y":"./vendor/"}}}', + ); + const vendorDir = path.join(functionDir, "vendor"); + yield* fs.makeDirectory(vendorDir, { recursive: true }); + const danglingLink = path.join(vendorDir, "dangling.ts"); + yield* fs.symlink(path.join(vendorDir, "does-not-exist.ts"), danglingLink); + + const error = yield* functionsDeploy(baseFlags).pipe(Effect.flip); + + if (!(error instanceof Error)) { + throw new Error(`unexpected error: ${String(error)}`); + } + // Matches the message `git show ec079c9b3:apps/cli/src/shared/functions/deploy.ts` + // produces for this exact fixture: Node's own raw ENOENT text from `stat`, not a + // reformatted `PlatformError` message. + expect(error.message).toBe(`ENOENT: no such file or directory, stat '${danglingLink}'`); + }).pipe(Effect.provide(layer), Effect.ensuring(removeTempRoot)); + }, + ); }); diff --git a/apps/cli/src/shared/functions/deploy.errors.ts b/apps/cli/src/shared/functions/deploy.errors.ts index 6b9198ca21..97e234eb09 100644 --- a/apps/cli/src/shared/functions/deploy.errors.ts +++ b/apps/cli/src/shared/functions/deploy.errors.ts @@ -1,52 +1,132 @@ -import { Data } from "effect"; +import { Schema } from "effect"; import { actionability, type CliErrorActionabilityDeclaration, ErrorActionabilityId, } from "../telemetry/error-actionability.ts"; -export class ConflictingFunctionDeployFlagsError extends Data.TaggedError( +export class ConflictingFunctionDeployFlagsError extends Schema.TaggedError()( "ConflictingFunctionDeployFlagsError", -)<{ - readonly message: string; -}> { + { + message: Schema.String, + }, +) { get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { return actionability.provideFlags; } } -export class InvalidFunctionDeploySlugError extends Data.TaggedError( +export class FunctionDeployJobsRequiresApiError extends Schema.TaggedError()( + "FunctionDeployJobsRequiresApiError", + { + message: Schema.String, + }, +) { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return actionability.provideFlags; + } +} + +export class InvalidFunctionDeploySlugError extends Schema.TaggedError()( "InvalidFunctionDeploySlugError", -)<{ - readonly message: string; -}> { + { + message: Schema.String, + }, +) { get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { return actionability.provideFlags; } } -export class NoFunctionsToDeployError extends Data.TaggedError("NoFunctionsToDeployError")<{ - readonly message: string; -}> { +export class NoFunctionsToDeployError extends Schema.TaggedError()( + "NoFunctionsToDeployError", + { + message: Schema.String, + }, +) { get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { return actionability.provideFlags; } } -export class FunctionDeployCancelledError extends Data.TaggedError("FunctionDeployCancelledError")<{ - readonly message: string; -}> { +export class FunctionDeployCancelledError extends Schema.TaggedError()( + "FunctionDeployCancelledError", + { + message: Schema.String, + }, +) { get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { return actionability.cancelled; } } -export class FunctionImportNotDirectoryError extends Data.TaggedError( +export class FunctionImportNotDirectoryError extends Schema.TaggedError()( "FunctionImportNotDirectoryError", -)<{ - readonly message: string; -}> { + { + message: Schema.String, + }, +) { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return actionability.invalidConfig; + } +} + +/** An asset, import map, or import-map target resolved outside its allowed root. */ +export class FunctionAssetOutsideRootError extends Schema.TaggedError()( + "FunctionAssetOutsideRootError", + { + message: Schema.String, + }, +) { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return actionability.invalidConfig; + } +} + +/** A configured entrypoint, static file, or import map resolved to a directory. */ +export class FunctionAssetIsDirectoryError extends Schema.TaggedError()( + "FunctionAssetIsDirectoryError", + { + message: Schema.String, + }, +) { get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { return actionability.invalidConfig; } } + +/** A `static_files` glob pattern matched no files on disk. */ +export class FunctionStaticPatternNoMatchError extends Schema.TaggedError()( + "FunctionStaticPatternNoMatchError", + { + message: Schema.String, + }, +) { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return actionability.invalidConfig; + } +} + +/** A `deno.json`/`deno.jsonc` or import map file failed to parse. */ +export class FunctionImportMapParseError extends Schema.TaggedError()( + "FunctionImportMapParseError", + { + message: Schema.String, + }, +) { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return actionability.invalidConfig; + } +} + +/** The Docker bundler container exited unsuccessfully. */ +export class FunctionBundleFailedError extends Schema.TaggedError()( + "FunctionBundleFailedError", + { + message: Schema.String, + }, +) { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return actionability.toolFailed; + } +} diff --git a/apps/cli/src/shared/functions/deploy.ts b/apps/cli/src/shared/functions/deploy.ts index ae2a57cbef..b991b994e2 100644 --- a/apps/cli/src/shared/functions/deploy.ts +++ b/apps/cli/src/shared/functions/deploy.ts @@ -1,6 +1,6 @@ import { brotliCompressSync, constants as zlibConstants } from "node:zlib"; -import { chmod, mkdir, mkdtemp, readFile, readdir, realpath, rm, stat } from "node:fs/promises"; -import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- pure, synchronous path-string math (no I/O), the same convention as command-internal/path-containment.ts and docker-ids.ts. +import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path"; import { URL } from "node:url"; import { FunctionResponse_Output, @@ -12,11 +12,13 @@ import { inferFunctionsManifest, type ResolvedFunctionConfig as ManifestFunctionConfig, } from "@supabase/config/effect"; -import { Duration, Effect, Option, Schema } from "effect"; +import { Cause, Config, Duration, Effect, FileSystem, Option, Ref, Schedule, Schema } from "effect"; +import type { PlatformError } from "effect/PlatformError"; import * as HttpBody from "effect/unstable/http/HttpBody"; import * as HttpClientError from "effect/unstable/http/HttpClientError"; import { promptYesNo } from "../../command-internal/prompt-yes-no.ts"; import { bitbucketCloneDir } from "../../command-internal/bitbucket-pipeline.ts"; +import { isPathContainedInRoot } from "../../command-internal/path-containment.ts"; import { CONTEXT_CANCELED_MESSAGE } from "../output/errors.ts"; import { Output } from "../output/output.service.ts"; import { bold } from "../../command-internal/colors.ts"; @@ -37,8 +39,14 @@ import { import { slimImagesEnabled } from "../services/slim-images.ts"; import { ConflictingFunctionDeployFlagsError, + FunctionAssetIsDirectoryError, + FunctionAssetOutsideRootError, + FunctionBundleFailedError, FunctionDeployCancelledError, + FunctionDeployJobsRequiresApiError, + FunctionImportMapParseError, FunctionImportNotDirectoryError, + FunctionStaticPatternNoMatchError, InvalidFunctionDeploySlugError, NoFunctionsToDeployError, } from "./deploy.errors.ts"; @@ -48,6 +56,9 @@ import { ensureDockerNamedVolume, ensureDockerNetwork, isDockerRunning, + type NativeFailure, + nativeFailure, + nativePlatformFailure, resolveDockerNetworkMode, resolveEdgeRuntimeVersion, resolveFunctionsDockerImage, @@ -367,16 +378,11 @@ function toApiRelativePath(cwd: string, hostPath: string) { return toSlash(relativePath.length > 0 ? relativePath : basename(resolved)); } -function isContainedPath(root: string, candidate: string) { - const relativePath = relative(resolve(root), resolve(candidate)); - return ( - relativePath === "" || - (!isAbsolute(relativePath) && relativePath !== ".." && !relativePath.startsWith(`..${sep}`)) - ); -} - +// Call sites always pass already-`realPath`-resolved, absolute candidates and +// roots, so the shared primitive's "both arguments must already be +// canonicalized" precondition always holds here. function isContainedInAnyPath(roots: ReadonlyArray, candidate: string) { - return roots.some((root) => isContainedPath(root, candidate)); + return roots.some((root) => isPathContainedInRoot(root, candidate)); } /** @@ -390,22 +396,31 @@ function hasParentPathSegment(relativePath: string) { .some((segment) => segment === ".."); } -async function realpathIfExists(pathname: string) { - try { - return await realpath(resolve(pathname)); - } catch (error) { - // ENOTDIR (a path routed through a file) is as nonexistent as ENOENT here. - if ( - error instanceof Error && - "code" in error && - (error.code === "ENOENT" || error.code === "ENOTDIR") - ) { - return resolve(pathname); - } - throw error; - } +function fsErrorCode(error: PlatformError): string | undefined { + const cause = error.reason.cause; + return cause instanceof Error && "code" in cause ? String(cause.code) : undefined; } +/** Curried `nativePlatformFailure` for `Effect.mapError`, labeling the failing path once. */ +const nativeFsError = (path: string) => (error: PlatformError) => + nativePlatformFailure(error, path); + +const realpathIfExists = Effect.fn("functions.deploy.realpathIfExists")(function* ( + fs: FileSystem.FileSystem, + pathname: string, +) { + return yield* fs.realPath(resolve(pathname)).pipe( + Effect.catchTag("PlatformError", (error) => { + // ENOTDIR (a path routed through a file) is as nonexistent as ENOENT here. + const code = fsErrorCode(error); + if (error.reason._tag === "NotFound" || code === "ENOTDIR") { + return Effect.succeed(resolve(pathname)); + } + return Effect.fail(nativePlatformFailure(error, pathname)); + }), + ); +}); + const resolveFunctionsSourceRoot = (projectRoot: string) => findGitRootPath(projectRoot).pipe(Effect.map(Option.getOrElse(() => resolve(projectRoot)))); @@ -540,23 +555,27 @@ function getObjectProperty(input: object, key: string): unknown { return Reflect.get(input, key); } -function readStringMap(input: unknown, fieldName: string): Record { +const readStringMap = Effect.fnUntraced(function* (input: unknown, fieldName: string) { if (input === undefined) { return {}; } if (typeof input !== "object" || input === null || Array.isArray(input)) { - throw new Error(`failed to parse import map: expected ${fieldName} to be an object`); + return yield* new FunctionImportMapParseError({ + message: `failed to parse import map: expected ${fieldName} to be an object`, + }); } const values: Record = {}; for (const [key, value] of Object.entries(input)) { if (typeof value !== "string") { - throw new Error(`failed to parse import map: expected ${fieldName}.${key} to be a string`); + return yield* new FunctionImportMapParseError({ + message: `failed to parse import map: expected ${fieldName}.${key} to be a string`, + }); } values[key] = value; } return values; -} +}); class ImportMapFile { readonly imports: Record; @@ -573,7 +592,7 @@ class ImportMapFile { this.importMapReference = importMapReference; } - static fromUnknown(input: unknown) { + static fromUnknown = Effect.fnUntraced(function* (input: unknown) { const imports: Record = {}; const scopes: Record> = {}; let importMapReference = ""; @@ -584,22 +603,24 @@ class ImportMapFile { importMapReference = importMap; } - Object.assign(imports, readStringMap(getObjectProperty(input, "imports"), "imports")); + Object.assign(imports, yield* readStringMap(getObjectProperty(input, "imports"), "imports")); const rawScopes = getObjectProperty(input, "scopes"); if (rawScopes === undefined) { return new ImportMapFile(imports, scopes, importMapReference); } if (typeof rawScopes !== "object" || rawScopes === null || Array.isArray(rawScopes)) { - throw new Error("failed to parse import map: expected scopes to be an object"); + return yield* new FunctionImportMapParseError({ + message: "failed to parse import map: expected scopes to be an object", + }); } for (const [scopeName, scopeValue] of Object.entries(rawScopes)) { - scopes[scopeName] = readStringMap(scopeValue, `scopes.${scopeName}`); + scopes[scopeName] = yield* readStringMap(scopeValue, `scopes.${scopeName}`); } } return new ImportMapFile(imports, scopes, importMapReference); - } + }); isReference() { return ( @@ -631,29 +652,60 @@ class ImportMapFile { } } -async function loadImportMapFile( - pathname: string, - onRead?: (pathname: string, contents: Uint8Array) => Promise, - seen = new Set(), -): Promise { - const resolvedPath = resolve(pathname); - if (seen.has(resolvedPath)) { - throw new Error(`cyclic import map reference: ${pathname}`); - } - seen.add(resolvedPath); - const contents = await readFile(pathname); - if (onRead !== undefined) { - await onRead(pathname, contents); - } - const parsed = JSON.parse(stripJsonComments(new TextDecoder().decode(contents))); - const importMap = ImportMapFile.fromUnknown(parsed).resolve(toSlash(pathname)); - if (isDenoConfigFile(pathname) && importMap.isReference()) { - const nestedPath = join(dirname(pathname), importMap.importMapReference); - return loadImportMapFile(nestedPath, onRead, seen); - } - return importMap; +/** + * Shared by every JSONC import-map read (`loadImportMapFile`, `resolveImportMapAllowedRoots`). + * Stays on raw `JSON.parse`, not `Schema.fromJsonString`: that combinator's own JSON-parse step + * (`SchemaGetter.parseJson`) discards the native `SyntaxError` and reports a fixed + * "a valid JSON string" message instead, which would change this error's existing text for a + * malformed file. + */ +function decodeJsoncBytes( + contents: Uint8Array, +): Effect.Effect { + return Effect.try({ + // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- see this function's own doc comment: Schema.fromJsonString discards the native SyntaxError this error's message needs + try: () => JSON.parse(stripJsonComments(new TextDecoder().decode(contents))), + catch: (cause) => + new FunctionImportMapParseError({ + message: cause instanceof Error ? cause.message : String(cause), + }), + }); } +// Explicit type annotation breaks the circular inference from the recursive +// `yield* loadImportMapFile(...)` call below. +const loadImportMapFile: ( + fs: FileSystem.FileSystem, + pathname: string, + onRead?: (pathname: string, contents: Uint8Array) => Effect.Effect, + seen?: Set, +) => Effect.Effect = + Effect.fnUntraced(function* ( + fs: FileSystem.FileSystem, + pathname: string, + onRead?: (pathname: string, contents: Uint8Array) => Effect.Effect, + seen: Set = new Set(), + ) { + const resolvedPath = resolve(pathname); + if (seen.has(resolvedPath)) { + return yield* new FunctionImportMapParseError({ + message: `cyclic import map reference: ${pathname}`, + }); + } + seen.add(resolvedPath); + const contents = yield* fs.readFile(pathname).pipe(Effect.mapError(nativeFsError(pathname))); + if (onRead !== undefined) { + yield* onRead(pathname, contents); + } + const parsed = yield* decodeJsoncBytes(contents); + const importMap = (yield* ImportMapFile.fromUnknown(parsed)).resolve(toSlash(pathname)); + if (isDenoConfigFile(pathname) && importMap.isReference()) { + const nestedPath = join(dirname(pathname), importMap.importMapReference); + return yield* loadImportMapFile(fs, nestedPath, onRead, seen); + } + return importMap; + }); + function substituteImportMapValue( mappings: Readonly>, specifier: string, @@ -726,13 +778,14 @@ function resolveImportSpecifier( return { path: resolved, substituted }; } -async function walkImportPaths( +const walkImportPaths = Effect.fnUntraced(function* ( + fs: FileSystem.FileSystem, importMap: ImportMapFile, srcPath: string, allowedRoots: ReadonlyArray, displayRoot: string, - onFile: (pathname: string, contents: Uint8Array) => Promise, - onWarning: (message: string) => Promise, + onFile: (pathname: string, contents: Uint8Array) => Effect.Effect, + onWarning: (message: string) => Effect.Effect, ) { const seen = new Set(); const queue = [toSlash(srcPath)]; @@ -744,33 +797,39 @@ async function walkImportPaths( } seen.add(current); - let contents: Uint8Array; - try { - const resolvedCurrent = await realpath(resolve(current)); + const maybeContents: Option.Option = yield* Effect.gen(function* () { + const resolvedCurrent = yield* fs.realPath(resolve(current)); if (!isContainedInAnyPath(allowedRoots, resolvedCurrent)) { - await onWarning(`WARN: Skipping import path outside source root: ${current}\n`); - continue; + yield* onWarning(`WARN: Skipping import path outside source root: ${current}\n`); + return Option.none(); } - contents = await readFile(resolvedCurrent); - } catch (error) { - if (error instanceof Error && "code" in error) { - if (error.code === "ENOENT") { - const message = `failed to read file: open ${toApiRelativePath(displayRoot, current)}: no such file or directory`; - await onWarning(`WARN: ${message}\n`); - continue; - } - // An ENOTDIR (import path routed through a file) gets a classified, user-facing message - // instead of an unhandled raw Node error, so telemetry books it as user-fixable config. - if (error.code === "ENOTDIR") { - throw new FunctionImportNotDirectoryError({ - message: `failed to read file: open ${toApiRelativePath(displayRoot, current)}: not a directory`, - }); - } - } - throw error; + return Option.some(yield* fs.readFile(resolvedCurrent)); + }).pipe( + Effect.catchTag("PlatformError", (error) => + Effect.gen(function* () { + if (error.reason._tag === "NotFound") { + const message = `failed to read file: open ${toApiRelativePath(displayRoot, current)}: no such file or directory`; + yield* onWarning(`WARN: ${message}\n`); + return Option.none(); + } + // An ENOTDIR (import path routed through a file) gets a classified, user-facing message + // instead of an unhandled raw Node error, so telemetry books it as user-fixable config. + if (fsErrorCode(error) === "ENOTDIR") { + return yield* new FunctionImportNotDirectoryError({ + message: `failed to read file: open ${toApiRelativePath(displayRoot, current)}: not a directory`, + }); + } + return yield* nativePlatformFailure(error, current); + }), + ), + ); + + if (Option.isNone(maybeContents)) { + continue; } + const contents = maybeContents.value; - await onFile(current, contents); + yield* onFile(current, contents); const text = new TextDecoder().decode(contents); importPathPattern.lastIndex = 0; for (const match of text.matchAll(importPathPattern)) { @@ -808,15 +867,15 @@ async function walkImportPaths( } const resolvedModule = resolve(modulePath); - const containmentPath = await realpathIfExists(resolvedModule); + const containmentPath = yield* realpathIfExists(fs, resolvedModule); if (!isContainedInAnyPath(allowedRoots, containmentPath)) { - await onWarning(`WARN: Skipping import path outside source root: ${modulePath}\n`); + yield* onWarning(`WARN: Skipping import path outside source root: ${modulePath}\n`); continue; } queue.push(toSlash(resolvedModule)); } } -} +}); function hasGlobMeta(pattern: string) { return pattern.includes("*") || pattern.includes("?") || pattern.includes("["); @@ -885,131 +944,161 @@ function globBaseDirectory(pattern: string) { return stableParts.join("/"); } -async function listPathsRecursive(root: string): Promise> { +// Node's native recursive `readdir` yields every intermediate directory alongside leaf files, +// relative to `root`. +const listPathsRecursive = Effect.fnUntraced(function* (fs: FileSystem.FileSystem, root: string) { const resolvedRoot = resolve(root); - const entries = await readdir(resolvedRoot, { withFileTypes: true }); - const paths: string[] = []; - for (const entry of entries) { - const pathname = join(resolvedRoot, entry.name); - paths.push(pathname); - if (entry.isDirectory()) { - paths.push(...(await listPathsRecursive(pathname))); - } - } - return paths; -} + const entries = yield* fs.readDirectory(resolvedRoot, { recursive: true }); + return entries.map((entry) => join(resolvedRoot, entry)); +}); + +const expandStaticPattern = Effect.fn("functions.deploy.expandStaticPattern")(function* ( + fs: FileSystem.FileSystem, + pattern: string, +) { + const noMatch = () => + new FunctionStaticPatternNoMatchError({ message: `no files matched pattern: ${pattern}` }); -async function expandStaticPattern(pattern: string): Promise> { if (!hasGlobMeta(pattern)) { - try { - await stat(pattern); - } catch { - throw new Error(`no files matched pattern: ${pattern}`); + const exists = yield* fs.stat(pattern).pipe( + Effect.as(true), + Effect.catchTag("PlatformError", () => Effect.succeed(false)), + ); + if (!exists) { + return yield* noMatch(); } return [pattern]; } const baseDir = globBaseDirectory(pattern); - const matcher = globToRegExp(toSlash(resolve(pattern))); - let candidates: ReadonlyArray; - try { - candidates = await listPathsRecursive(baseDir); - } catch (error) { - if (error instanceof Error && "code" in error && error.code === "ENOENT") { - throw new Error(`no files matched pattern: ${pattern}`); - } - throw error; - } + // `globToRegExp` builds its result with `new RegExp(...)`, which throws a native `SyntaxError` + // for an invalid bracket expression (e.g. `[z-a]`); keep that as a typed failure the recovery + // below can catch, not a defect that would bypass it. + const matcher = yield* Effect.try({ + try: () => globToRegExp(toSlash(resolve(pattern))), + catch: (cause) => nativeFailure(cause), + }); + const candidates = yield* listPathsRecursive(fs, baseDir).pipe( + Effect.catchTag( + "PlatformError", + (error): Effect.Effect => + error.reason._tag === "NotFound" + ? Effect.fail(noMatch()) + : Effect.fail(nativePlatformFailure(error, baseDir)), + ), + ); const matches = candidates.filter((candidate) => matcher.test(toSlash(resolve(candidate)))); if (matches.length === 0) { - throw new Error(`no files matched pattern: ${pattern}`); + return yield* noMatch(); } return matches; -} +}); -async function forEachLocalImportMapTarget( +const forEachLocalImportMapTarget = Effect.fnUntraced(function* ( importMap: ImportMapFile, - onTarget: (pathname: string, kind: "import" | "scope") => Promise, + onTarget: (pathname: string, kind: "import" | "scope") => Effect.Effect, ) { for (const target of Object.values(importMap.imports)) { if (isRemoteImportTarget(target)) { continue; } - await onTarget(target, "import"); + yield* onTarget(target, "import"); } for (const scope of Object.values(importMap.scopes)) { for (const target of Object.values(scope)) { if (isRemoteImportTarget(target)) { continue; } - await onTarget(target, "scope"); + yield* onTarget(target, "scope"); } } -} +}); -async function walkLocalImportMapTargetImports( +const walkLocalImportMapTargetImports = Effect.fnUntraced(function* ( + fs: FileSystem.FileSystem, importMap: ImportMapFile, pathname: string, allowedRoots: ReadonlyArray, displayRoot: string, - onFile: (pathname: string, contents: Uint8Array) => Promise, - onWarning: (message: string) => Promise, + onFile: (pathname: string, contents: Uint8Array) => Effect.Effect, + onWarning: (message: string) => Effect.Effect, ) { - if ((await stat(pathname)).isDirectory()) { + const info = yield* fs.stat(pathname).pipe(Effect.mapError(nativeFsError(pathname))); + if (info.type === "Directory") { return; } - await walkImportPaths(importMap, pathname, allowedRoots, displayRoot, onFile, onWarning); -} + yield* walkImportPaths(fs, importMap, pathname, allowedRoots, displayRoot, onFile, onWarning); +}); -async function isFile(pathname: string): Promise { - try { - return (await stat(pathname)).isFile(); - } catch { - return false; - } -} +const isFile = Effect.fnUntraced(function* (fs: FileSystem.FileSystem, pathname: string) { + return yield* fs.stat(pathname).pipe( + Effect.map((info) => info.type === "File"), + Effect.catchTag("PlatformError", () => Effect.succeed(false)), + ); +}); -async function resolveImportMapAllowedRoots(projectRoot: string, importMapPath: string) { - const realProjectRoot = await realpath(projectRoot); +const resolveImportMapAllowedRoots = Effect.fnUntraced(function* ( + fs: FileSystem.FileSystem, + projectRoot: string, + importMapPath: string, +) { + const realProjectRoot = yield* fs + .realPath(projectRoot) + .pipe(Effect.mapError(nativeFsError(projectRoot))); const allowedRoots = [realProjectRoot]; if (importMapPath.length === 0) { return allowedRoots; } - const realImportMapPath = await realpath(importMapPath); - if (!isContainedPath(realProjectRoot, realImportMapPath)) { + const realImportMapPath = yield* fs + .realPath(importMapPath) + .pipe(Effect.mapError(nativeFsError(importMapPath))); + if (!isPathContainedInRoot(realProjectRoot, realImportMapPath)) { allowedRoots.push(dirname(realImportMapPath)); } if (isDenoConfigFile(importMapPath)) { - const contents = await readFile(importMapPath); - const parsed = JSON.parse(stripJsonComments(new TextDecoder().decode(contents))); - const importMap = ImportMapFile.fromUnknown(parsed); + const contents = yield* fs + .readFile(importMapPath) + .pipe(Effect.mapError(nativeFsError(importMapPath))); + const parsed = yield* decodeJsoncBytes(contents); + const importMap = yield* ImportMapFile.fromUnknown(parsed); if (importMap.importMapReference.length > 0) { - const referencedImportMapPath = await realpath( - join(dirname(importMapPath), importMap.importMapReference), - ); - if (!isContainedPath(realProjectRoot, referencedImportMapPath)) { + const referencedPath = join(dirname(importMapPath), importMap.importMapReference); + const referencedImportMapPath = yield* fs + .realPath(referencedPath) + .pipe(Effect.mapError(nativeFsError(referencedPath))); + if (!isPathContainedInRoot(realProjectRoot, referencedImportMapPath)) { allowedRoots.push(dirname(referencedImportMapPath)); } } } return allowedRoots; -} +}); -async function writeSourceDeployForm( +const writeSourceDeployForm = Effect.fn("functions.deploy.writeSourceDeployForm")(function* ( + fs: FileSystem.FileSystem, sourceRoot: string, workdir: string, config: ResolvedDeployFunctionConfig, - metadata: SourceDeployMetadata, outputRaw: (text: string) => Effect.Effect, ) { - const form = new FormData(); - form.append("metadata", JSON.stringify(metadata)); - const realSourceRoot = await realpath(sourceRoot); - const importMapAllowedRoots = await resolveImportMapAllowedRoots(sourceRoot, config.importMap); + const files: Array = []; + const hasImportMap = config.importMap.length > 0; + const realSourceRoot = yield* fs + .realPath(sourceRoot) + .pipe(Effect.mapError(nativeFsError(sourceRoot))); + const importMapAllowedRoots = yield* resolveImportMapAllowedRoots( + fs, + sourceRoot, + config.importMap, + ); const uploadedAssets = new Set(); - const appendAsset = async (pathname: string, contents: Uint8Array, realPathname: string) => { + const appendAsset = Effect.fnUntraced(function* ( + pathname: string, + contents: Uint8Array, + realPathname: string, + ) { if (uploadedAssets.has(realPathname)) { return; } @@ -1018,130 +1107,164 @@ async function writeSourceDeployForm( // `deployViaApi`. const relativePath = toApiRelativePath(workdir, pathname); if (hasParentPathSegment(relativePath)) { - throw new Error(`failed to read file: open ${relativePath}: invalid argument`); + return yield* new FunctionAssetOutsideRootError({ + message: `failed to read file: open ${relativePath}: invalid argument`, + }); } - await Effect.runPromise(outputRaw(`Uploading asset (${config.slug}): ${relativePath}\n`)); - form.append("file", new File([contents], relativePath)); - }; + yield* outputRaw(`Uploading asset (${config.slug}): ${relativePath}\n`); + files.push(new File([contents], relativePath)); + }); - const uploadAsset = async (pathname: string, contents: Uint8Array) => { - const realPathname = await realpath(pathname); - if (!isContainedPath(realSourceRoot, realPathname)) { - throw new Error(`refusing to upload asset outside source root: ${pathname}`); + const uploadAsset = Effect.fnUntraced(function* (pathname: string, contents: Uint8Array) { + const realPathname = yield* fs + .realPath(pathname) + .pipe(Effect.mapError(nativeFsError(pathname))); + if (!isPathContainedInRoot(realSourceRoot, realPathname)) { + return yield* new FunctionAssetOutsideRootError({ + message: `refusing to upload asset outside source root: ${pathname}`, + }); } - await appendAsset(pathname, contents, realPathname); - }; + yield* appendAsset(pathname, contents, realPathname); + }); - const uploadImportMapAsset = async (pathname: string, contents: Uint8Array) => { - const realPathname = await realpath(pathname); + const uploadImportMapAsset = Effect.fnUntraced(function* ( + pathname: string, + contents: Uint8Array, + ) { + const realPathname = yield* fs + .realPath(pathname) + .pipe(Effect.mapError(nativeFsError(pathname))); if (!isContainedInAnyPath(importMapAllowedRoots, realPathname)) { - throw new Error(`refusing to upload import map outside allowed roots: ${pathname}`); + return yield* new FunctionAssetOutsideRootError({ + message: `refusing to upload import map outside allowed roots: ${pathname}`, + }); } - await appendAsset(pathname, contents, realPathname); - }; + yield* appendAsset(pathname, contents, realPathname); + }); - const uploadImportMapTargetAsset = async (pathname: string, contents: Uint8Array) => { - const realPathname = await realpath(pathname); + const uploadImportMapTargetAsset = Effect.fnUntraced(function* ( + pathname: string, + contents: Uint8Array, + ) { + const realPathname = yield* fs + .realPath(pathname) + .pipe(Effect.mapError(nativeFsError(pathname))); if (!isContainedInAnyPath(importMapAllowedRoots, realPathname)) { - await Effect.runPromise( - outputRaw(`WARN: Skipping import path outside source root: ${pathname}\n`), - ); + yield* outputRaw(`WARN: Skipping import path outside source root: ${pathname}\n`); return; } - await appendAsset(pathname, contents, realPathname); - }; + yield* appendAsset(pathname, contents, realPathname); + }); - const uploadScopeTarget = async (pathname: string) => { - let resolvedPath: string; - let pathInfo: Awaited>; - try { - resolvedPath = await realpath(pathname); - pathInfo = await stat(pathname); - } catch (error) { - if (error instanceof Error && "code" in error && error.code === "ENOTDIR") { - await Effect.runPromise( - outputRaw(`WARN: Skipping import map target that is not a directory: ${pathname}\n`), - ); - return; - } - throw error; + const uploadScopeTarget = Effect.fnUntraced(function* (pathname: string) { + const outcome = yield* fs.realPath(pathname).pipe( + Effect.flatMap((resolvedPath) => + fs.stat(pathname).pipe(Effect.map((pathInfo) => Option.some({ resolvedPath, pathInfo }))), + ), + Effect.catchTag("PlatformError", (error) => + fsErrorCode(error) === "ENOTDIR" + ? Effect.succeed( + Option.none<{ + readonly resolvedPath: string; + readonly pathInfo: FileSystem.File.Info; + }>(), + ) + : Effect.fail(nativePlatformFailure(error, pathname)), + ), + ); + if (Option.isNone(outcome)) { + yield* outputRaw(`WARN: Skipping import map target that is not a directory: ${pathname}\n`); + return; } + const { resolvedPath, pathInfo } = outcome.value; if (!isContainedInAnyPath(importMapAllowedRoots, resolvedPath)) { - await Effect.runPromise( - outputRaw(`WARN: Skipping import path outside source root: ${pathname}\n`), - ); + yield* outputRaw(`WARN: Skipping import path outside source root: ${pathname}\n`); return; } - if (!pathInfo.isDirectory()) { - await uploadImportMapTargetAsset(pathname, await readFile(pathname)); - await walkLocalImportMapTargetImports( + if (pathInfo.type !== "Directory") { + const contents = yield* fs.readFile(pathname).pipe(Effect.mapError(nativeFsError(pathname))); + yield* uploadImportMapTargetAsset(pathname, contents); + yield* walkLocalImportMapTargetImports( + fs, importMap, pathname, importMapAllowedRoots, workdir, uploadImportMapTargetAsset, - async (message) => { - await Effect.runPromise(outputRaw(message)); - }, + outputRaw, ); return; } - const nestedPaths = await listPathsRecursive(pathname); + const nestedPaths = yield* listPathsRecursive(fs, pathname).pipe( + Effect.mapError(nativeFsError(pathname)), + ); for (const nestedPath of nestedPaths) { - if ((await stat(nestedPath)).isDirectory()) { + const nestedInfo = yield* fs + .stat(nestedPath) + .pipe(Effect.mapError(nativeFsError(nestedPath))); + if (nestedInfo.type === "Directory") { continue; } - const resolvedNestedPath = await realpath(nestedPath); + const resolvedNestedPath = yield* fs + .realPath(nestedPath) + .pipe(Effect.mapError(nativeFsError(nestedPath))); if (!isContainedInAnyPath(importMapAllowedRoots, resolvedNestedPath)) { - await Effect.runPromise( - outputRaw(`WARN: Skipping import path outside source root: ${nestedPath}\n`), - ); + yield* outputRaw(`WARN: Skipping import path outside source root: ${nestedPath}\n`); continue; } - await uploadImportMapTargetAsset(nestedPath, await readFile(nestedPath)); + const nestedContents = yield* fs + .readFile(nestedPath) + .pipe(Effect.mapError(nativeFsError(nestedPath))); + yield* uploadImportMapTargetAsset(nestedPath, nestedContents); } - }; + }); - if (metadata.import_map_path !== undefined && metadata.import_map_path.length > 0) { - await loadImportMapFile(config.importMap, uploadImportMapAsset); + if (hasImportMap) { + yield* loadImportMapFile(fs, config.importMap, uploadImportMapAsset); } for (const pattern of config.staticFiles) { - let files: ReadonlyArray; - try { - files = await expandStaticPattern(pattern); - } catch (error) { - await Effect.runPromise( - outputRaw(`WARN: ${error instanceof Error ? error.message : String(error)}\n`), - ); + // Every typed expansion failure (no match, an unreadable directory, ...) is warned and + // skipped. Interruptions and defects are untouched — only the two declared tags are caught. + const warnAndSkip = (error: { readonly message: string }) => + outputRaw(`WARN: ${error.message}\n`).pipe(Effect.as(undefined)); + const files: ReadonlyArray | undefined = yield* expandStaticPattern(fs, pattern).pipe( + Effect.catchTags({ + FunctionStaticPatternNoMatchError: warnAndSkip, + NativeFailure: warnAndSkip, + }), + ); + if (files === undefined) { continue; } for (const pathname of files) { - if ((await stat(pathname)).isDirectory()) { - throw new Error(`file path is a directory: ${pathname}`); + const info = yield* fs.stat(pathname).pipe(Effect.mapError(nativeFsError(pathname))); + if (info.type === "Directory") { + return yield* new FunctionAssetIsDirectoryError({ + message: `file path is a directory: ${pathname}`, + }); } - await uploadAsset(pathname, await readFile(pathname)); + const contents = yield* fs.readFile(pathname).pipe(Effect.mapError(nativeFsError(pathname))); + yield* uploadAsset(pathname, contents); } } - const importMap = - metadata.import_map_path !== undefined && metadata.import_map_path.length > 0 - ? await loadImportMapFile(config.importMap) - : new ImportMapFile(); - await walkImportPaths( + const importMap = hasImportMap + ? yield* loadImportMapFile(fs, config.importMap) + : new ImportMapFile(); + yield* walkImportPaths( + fs, importMap, config.entrypoint, [realSourceRoot], workdir, uploadAsset, - async (message) => { - await Effect.runPromise(outputRaw(message)); - }, + outputRaw, ); - await forEachLocalImportMapTarget(importMap, uploadScopeTarget); + yield* forEachLocalImportMapTarget(importMap, uploadScopeTarget); - return form; -} + return files; +}); /** * Server-recorded metadata paths are anchored at the workdir, with forward slashes regardless of @@ -1209,7 +1332,7 @@ function sanitizeDockerBinds( type DockerBindsOptions = { readonly additionalModuleRoots?: ReadonlyArray; - readonly onWarning?: (message: string) => Promise; + readonly onWarning?: (message: string) => Effect.Effect; readonly skipMissingImportMapTargets?: boolean; /** Resolved marker presence, including an explicitly empty project value. */ readonly bitbucketCloneDirDefined?: boolean; @@ -1224,38 +1347,40 @@ export const buildDockerBinds = ( ) => resolveFunctionsSourceRoot(resolve(functionsDir, "..", "..")).pipe( Effect.flatMap((sourceRoot) => - Effect.promise(() => - buildDockerBindsWithin(sourceRoot, projectId, functionsDir, outputDir, config, options), + Effect.flatMap(FileSystem.FileSystem, (fs) => + buildDockerBindsWithin(fs, sourceRoot, projectId, functionsDir, outputDir, config, options), ), ), ); -async function buildDockerBindsWithin( +const buildDockerBindsWithin = Effect.fn("functions.deploy.buildDockerBinds")(function* ( + fs: FileSystem.FileSystem, sourceRoot: string, projectId: string, functionsDir: string, outputDir: string, config: ResolvedDeployFunctionConfig, options: DockerBindsOptions, -): Promise> { +) { const hostFunctionsDir = resolve(functionsDir); const hostOutputDir = resolve(outputDir); - const realSourceRoot = await realpath(sourceRoot); + const realSourceRoot = yield* fs + .realPath(sourceRoot) + .pipe(Effect.mapError(nativeFsError(sourceRoot))); + const resolvedAdditionalRoots = yield* Effect.forEach( + options.additionalModuleRoots ?? [], + (root) => fs.realPath(root).pipe(Effect.option), + { concurrency: "unbounded" }, + ); const moduleRoots = [ realSourceRoot, - ...( - await Promise.all( - (options.additionalModuleRoots ?? []).map(async (root) => { - try { - return await realpath(root); - } catch { - return undefined; - } - }), - ) - ).flatMap((root) => (root === undefined ? [] : [root])), + ...resolvedAdditionalRoots.flatMap((root) => (Option.isSome(root) ? [root.value] : [])), ]; - const importMapAllowedRoots = await resolveImportMapAllowedRoots(sourceRoot, config.importMap); + const importMapAllowedRoots = yield* resolveImportMapAllowedRoots( + fs, + sourceRoot, + config.importMap, + ); const binds: DockerBind[] = [ { hostPath: hostFunctionsDir, @@ -1283,11 +1408,18 @@ async function buildDockerBindsWithin( }); } - const warn = options.onWarning ?? (async () => {}); + const onWarning = options.onWarning; + const warn = (message: string) => (onWarning === undefined ? Effect.void : onWarning(message)); const extraBinds: DockerBind[] = []; const explicitScopeBinds = new Map(); - const appendBindWithinRoots = async (roots: ReadonlyArray, pathname: string) => { - const hostPath = await realpath(pathname); + // Left as an untyped `PlatformError`, not wrapped into `NativeFailure`: the only caller that + // inspects a failure here — the scope-target walk below — classifies ENOTDIR/ENOENT off the + // platform tag. + const appendBindWithinRoots = Effect.fnUntraced(function* ( + roots: ReadonlyArray, + pathname: string, + ) { + const hostPath = yield* fs.realPath(pathname); const contained = isContainedInAnyPath(roots, hostPath); if (contained) { extraBinds.push({ @@ -1298,21 +1430,19 @@ async function buildDockerBindsWithin( }); } return { hostPath, contained }; - }; - const appendProjectBind = async (pathname: string, _contents: Uint8Array) => { - await appendBindWithinRoots([realSourceRoot], pathname); - }; - const appendModuleBind = async (pathname: string, _contents: Uint8Array) => { - await appendBindWithinRoots(moduleRoots, pathname); - }; - const appendImportMapBind = async (pathname: string, _contents: Uint8Array) => { - await appendBindWithinRoots(importMapAllowedRoots, pathname); - }; + }); + const appendProjectBind = (pathname: string, _contents: Uint8Array) => + appendBindWithinRoots([realSourceRoot], pathname).pipe(Effect.asVoid); + const appendModuleBind = (pathname: string, _contents: Uint8Array) => + appendBindWithinRoots(moduleRoots, pathname).pipe(Effect.asVoid); + const appendImportMapBind = (pathname: string, _contents: Uint8Array) => + appendBindWithinRoots(importMapAllowedRoots, pathname).pipe(Effect.asVoid); const importMap = config.importMap.length > 0 - ? await loadImportMapFile(config.importMap, appendImportMapBind) + ? yield* loadImportMapFile(fs, config.importMap, appendImportMapBind) : new ImportMapFile(); - await walkImportPaths( + yield* walkImportPaths( + fs, importMap, config.entrypoint, moduleRoots, @@ -1320,10 +1450,10 @@ async function buildDockerBindsWithin( appendModuleBind, warn, ); - await forEachLocalImportMapTarget(importMap, async (target, kind) => { - try { - const { hostPath, contained } = await appendBindWithinRoots(importMapAllowedRoots, target); - const isDirectory = (await stat(target)).isDirectory(); + yield* forEachLocalImportMapTarget(importMap, (target, kind) => + Effect.gen(function* () { + const { hostPath, contained } = yield* appendBindWithinRoots(importMapAllowedRoots, target); + const isDirectory = (yield* fs.stat(target)).type === "Directory"; if (!contained && kind === "scope") { const scopeBind: DockerBind = { hostPath, @@ -1336,44 +1466,46 @@ async function buildDockerBindsWithin( if (isDirectory) { return; } - await walkLocalImportMapTargetImports( + yield* walkLocalImportMapTargetImports( + fs, importMap, target, importMapAllowedRoots, sourceRoot, appendImportMapBind, - async () => {}, + () => Effect.void, ); - } catch (error) { - if (error instanceof Error && "code" in error) { + }).pipe( + Effect.catchTag("PlatformError", (error) => { // ENOTDIR (a trailing-slash value routed through a file) is never a // walkable target regardless of caller: an import that actually // reaches through that file still fails via the walker's // FunctionImportNotDirectoryError. - if (error.code === "ENOTDIR") { - await warn(`WARN: Skipping import map target that is not a directory: ${target}\n`); - return; + if (fsErrorCode(error) === "ENOTDIR") { + return warn(`WARN: Skipping import map target that is not a directory: ${target}\n`); } - if (options.skipMissingImportMapTargets === true && error.code === "ENOENT") { - await warn(`WARN: Skipping missing import map target: ${target}\n`); - return; + if (options.skipMissingImportMapTargets === true && error.reason._tag === "NotFound") { + return warn(`WARN: Skipping missing import map target: ${target}\n`); } - } - throw error; - } - }); + return Effect.fail(nativePlatformFailure(error, target)); + }), + ), + ); for (const pattern of config.staticFiles) { - let files: ReadonlyArray; - try { - files = await expandStaticPattern(pattern); - } catch { + // Any expansion failure (no match, unreadable directory, ...) is silently skipped here, + // unlike the API-upload path's warned skip. + const files = yield* expandStaticPattern(fs, pattern).pipe(Effect.option); + if (Option.isNone(files)) { continue; } - for (const pathname of files) { - if ((await stat(pathname)).isDirectory()) { - throw new Error(`file path is a directory: ${pathname}`); + for (const pathname of files.value) { + const info = yield* fs.stat(pathname).pipe(Effect.mapError(nativeFsError(pathname))); + if (info.type === "Directory") { + return yield* new FunctionAssetIsDirectoryError({ + message: `file path is a directory: ${pathname}`, + }); } - await appendProjectBind(pathname, new Uint8Array()); + yield* appendProjectBind(pathname, new Uint8Array()); } } @@ -1388,29 +1520,31 @@ async function buildDockerBindsWithin( } occupiedContainerPaths.add(bind.containerPath); uniqueScopeBinds.push(bind); - await warn( + yield* warn( `WARN: Mounting import map scope target outside the project root: ${bind.hostPath}\n`, ); } return [...binds, ...sanitizedExtraBinds, ...uniqueScopeBinds]; -} +}); function shouldUseDenoJsonDiscovery(entrypoint: string, importMap: string) { return isDenoConfigFile(importMap) && dirname(importMap) === dirname(entrypoint); } -async function shouldUsePackageJsonDiscovery(entrypoint: string, importMap: string) { +const shouldUsePackageJsonDiscovery = Effect.fnUntraced(function* ( + fs: FileSystem.FileSystem, + entrypoint: string, + importMap: string, +) { if (importMap.length > 0) { return false; } - try { - await stat(join(dirname(entrypoint), "package.json")); - return true; - } catch { - return false; - } -} + return yield* fs.stat(join(dirname(entrypoint), "package.json")).pipe( + Effect.as(true), + Effect.catchTag("PlatformError", () => Effect.succeed(false)), + ); +}); interface BundleFunctionWithDockerOptions { readonly projectId: string; @@ -1439,22 +1573,24 @@ const bundleFunctionWithDocker = Effect.fn("functions.deploy.bundleWithDocker")( } = options; const bitbucketCloneDirDefined = Option.isSome(yield* bitbucketCloneDir(projectEnvValues)); const output = yield* Output; + const fs = yield* FileSystem.FileSystem; yield* output.raw(`Bundling Function: ${styleEmphasis(config.slug)}\n`, "stderr"); const outputRoot = resolve(functionsDir, "..", ".temp"); - yield* Effect.tryPromise(() => mkdir(outputRoot, { recursive: true })); + yield* fs + .makeDirectory(outputRoot, { recursive: true }) + .pipe(Effect.mapError(nativeFsError(outputRoot))); // A generator `finally` does not run when a yielded effect fails or is interrupted. return yield* Effect.acquireUseRelease( - Effect.tryPromise(() => mkdtemp(join(outputRoot, `.supabase-output-${config.slug}-`))), + fs + .makeTempDirectory({ directory: outputRoot, prefix: `.supabase-output-${config.slug}-` }) + .pipe(Effect.mapError(nativeFsError(outputRoot))), (outputDir) => Effect.gen(function* () { // Go passes 0777 to MkdirAll, which Windows ignores. Calling chmod separately // adds an NTFS WRITE_ATTRIBUTES requirement that the Go CLI does not have. if (shouldChmodBundleOutputDirectory(process.platform)) { - yield* Effect.tryPromise({ - try: () => chmod(outputDir, 0o777), - catch: (cause) => (cause instanceof Error ? cause : new Error(String(cause))), - }); + yield* fs.chmod(outputDir, 0o777).pipe(Effect.mapError(nativeFsError(outputDir))); } const outputPath = join(outputDir, "output.eszip"); // `edgeRuntimeImage` applies the tag verbatim — a `.temp/edge-runtime-version` pin flows @@ -1462,7 +1598,7 @@ const bundleFunctionWithDocker = Effect.fn("functions.deploy.bundleWithDocker")( const rawImage = edgeRuntimeImage(edgeRuntimeVersion, yield* slimImagesEnabled); const binds = yield* buildDockerBinds(projectId, functionsDir, outputDir, config, { bitbucketCloneDirDefined, - onWarning: (message) => Effect.runPromise(output.raw(message, "stderr")), + onWarning: (message) => output.raw(message, "stderr"), }); // Resolved per function rather than hoisted out of the loop (unlike `download.ts`'s // `PulledEdgeRuntimeImage`): the first resolve failure aborts the loop, and the only added @@ -1476,11 +1612,7 @@ const bundleFunctionWithDocker = Effect.fn("functions.deploy.bundleWithDocker")( ); const env: Array = []; - if ( - !(yield* Effect.promise(() => - shouldUsePackageJsonDiscovery(config.entrypoint, config.importMap), - )) - ) { + if (!(yield* shouldUsePackageJsonDiscovery(fs, config.entrypoint, config.importMap))) { env.push("DENO_NO_PACKAGE_JSON=1"); } env.push(...dockerNpmEnv()); @@ -1501,7 +1633,8 @@ const bundleFunctionWithDocker = Effect.fn("functions.deploy.bundleWithDocker")( for (const staticFile of config.staticFiles) { containerArgs.push("--static", toDockerPath(staticFile, { resolve })); } - if (verbose || process.env["DEBUG"] === "true") { + const debugEnv = yield* Config.option(Config.string("DEBUG")); + if (verbose || Option.contains(debugEnv, "true")) { containerArgs.push("--verbose"); } @@ -1526,18 +1659,19 @@ const bundleFunctionWithDocker = Effect.fn("functions.deploy.bundleWithDocker")( onStderr: (chunk) => output.raw(chunk, "stderr"), }); if (result.exitCode !== 0) { - return yield* Effect.fail( - new Error(`failed to bundle function: exit ${result.exitCode}`), - ); + return yield* new FunctionBundleFailedError({ + message: `failed to bundle function: exit ${result.exitCode}`, + }); } - const eszip = yield* Effect.tryPromise({ - try: () => readFile(outputPath), - catch: (error) => - new Error( - `failed to open eszip: ${error instanceof Error ? error.message : String(error)}`, - ), - }); + const eszip = yield* fs.readFile(outputPath).pipe( + Effect.mapError( + (error) => + new FunctionBundleFailedError({ + message: `failed to open eszip: ${nativePlatformFailure(error, outputPath).message}`, + }), + ), + ); const compressed = new Uint8Array( Buffer.concat([ Buffer.from(COMPRESSED_ESZIP_MAGIC), @@ -1558,69 +1692,75 @@ const bundleFunctionWithDocker = Effect.fn("functions.deploy.bundleWithDocker")( } satisfies BundledFunction; }), (outputDir) => - Effect.tryPromise(() => rm(outputDir, { recursive: true, force: true })).pipe( - Effect.orElseSucceed(() => undefined), - ), + fs + .remove(outputDir, { recursive: true, force: true }) + .pipe(Effect.orElseSucceed(() => undefined)), ); }); +// Transient-API bound shared by the list/bulk-update retry policies below: 3 retries (4 attempts +// total) with a 1s-base exponential backoff. +const transientApiRetrySchedule = Schedule.exponential("1 second").pipe( + Schedule.upTo({ times: 3 }), +); + +// Same 3-retry bound as `transientApiRetrySchedule`, with a 500ms base for `upsertBundledFunction`. +const upsertRetrySchedule = Schedule.exponential("500 millis").pipe(Schedule.upTo({ times: 3 })); + +/** Retries a transport-ish failure or a 5xx/429 status; a decode failure or other 4xx is terminal. */ +function isRetryableFunctionsApiError( + error: + | FunctionsApiStatusError + | FunctionsApiTransportError + | SupabaseApiInputError + | HttpBody.HttpBodyError, +): boolean { + return ( + !(error instanceof FunctionsApiStatusError) || + (error.decode !== true && (error.status >= 500 || error.status === 429)) + ); +} + const listRemoteFunctions = Effect.fn("functions.deploy.listRemoteFunctions")(function* ( api: ApiClient, projectRef: string, ) { - let lastError: Error | FunctionsApiStatusError | undefined; - for (let attempt = 0; attempt <= 3; attempt += 1) { - const result = yield* api + const attempt = Effect.gen(function* () { + const response = yield* api .executeRaw(operationDefinitions.v1ListAllFunctions, { ref: projectRef }) - .pipe( - Effect.map((response) => ({ success: true as const, response })), - Effect.catch((error) => - Effect.succeed({ - success: false as const, - error: mapTransportError("failed to list functions", error), + .pipe(Effect.mapError((error) => mapTransportError("failed to list functions", error))); + const body = yield* response.text.pipe(Effect.orElseSucceed(() => "")); + if (response.status === 200) { + // A 200 whose body is not the expected JSON is an API-response problem, + // not a transport failure — surface it via FunctionsApiStatusError so it + // classifies as api_status rather than network. + return yield* Effect.try({ + // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- Schema.fromJsonString discards the native SyntaxError on malformed JSON (see decodeJsoncBytes); this raw JSON.parse keeps that message for the decode-error branch below + try: () => decodeFunctionListResponse(JSON.parse(body)), + catch: (error) => + new FunctionsApiStatusError({ + status: response.status, + message: `failed to read functions list: ${error instanceof Error ? error.message : String(error)}`, + decode: true, }), - ), - ); - - if (result.success) { - const body = yield* result.response.text.pipe(Effect.orElseSucceed(() => "")); - if (result.response.status === 200) { - // A 200 whose body is not the expected JSON is an API-response problem, - // not a transport failure — surface it via FunctionsApiStatusError so it - // classifies as api_status rather than network. - return yield* Effect.try({ - try: () => decodeFunctionListResponse(JSON.parse(body)), - catch: (error) => - new FunctionsApiStatusError({ - status: result.response.status, - message: `failed to read functions list: ${error instanceof Error ? error.message : String(error)}`, - decode: true, - }), - }); - } - lastError = new FunctionsApiStatusError({ - status: result.response.status, - message: `unexpected list functions status ${result.response.status}: ${body}`, }); - if (result.response.status < 500 && result.response.status !== 429) { - return yield* Effect.fail(lastError); - } - } else { - lastError = result.error; } + return yield* new FunctionsApiStatusError({ + status: response.status, + message: `unexpected list functions status ${response.status}: ${body}`, + }); + }); - if (attempt < 3) { - yield* Effect.sleep(Duration.millis(1_000 * 2 ** attempt)); - } - } - return yield* Effect.fail(lastError ?? new Error("failed to list functions")); + return yield* attempt.pipe( + Effect.retry({ schedule: transientApiRetrySchedule, while: isRetryableFunctionsApiError }), + ); }); function headerValue(headers: Readonly>, name: string) { return headers[name.toLowerCase()] ?? headers[name]; } -function parseRateLimitDelay(value: string | undefined): number | undefined { +function parseRateLimitDelay(value: string | undefined, nowMillis: number): number | undefined { if (value === undefined || value.length === 0) { return undefined; } @@ -1630,18 +1770,20 @@ function parseRateLimitDelay(value: string | undefined): number | undefined { } const timestamp = Date.parse(value); if (!Number.isNaN(timestamp)) { - return Math.max(timestamp - Date.now(), 0); + return Math.max(timestamp - nowMillis, 0); } return undefined; } +/** `nowMillis` comes from the caller's `Schedule`/`Clock` metadata, never read from the system clock here. */ function rateLimitDelayMillis( headers: Readonly>, attempt: number, + nowMillis: number, ) { return ( - parseRateLimitDelay(headerValue(headers, "retry-after")) ?? - parseRateLimitDelay(headerValue(headers, "x-ratelimit-reset")) ?? + parseRateLimitDelay(headerValue(headers, "retry-after"), nowMillis) ?? + parseRateLimitDelay(headerValue(headers, "x-ratelimit-reset"), nowMillis) ?? 1_000 * 2 ** Math.min(attempt, 5) ); } @@ -1650,30 +1792,40 @@ function rateLimitDelayText(milliseconds: number) { return `${Math.round(milliseconds / 1_000)}s`; } +interface RateLimitableResponse { + readonly status: number; + readonly headers: Readonly>; + readonly body: Effect.Effect; +} + const rateLimitedRequest = Effect.fnUntraced(function* ( action: string, - request: () => Effect.Effect< - { - readonly status: number; - readonly headers: Readonly>; - readonly body: Effect.Effect; - }, - Error - >, + request: () => Effect.Effect, Error>, ) { const output = yield* Output; - for (let attempt = 0; ; attempt += 1) { - const response = yield* request(); - if (response.status !== 429 || attempt >= DEPLOY_RATE_LIMIT_MAX_RETRIES) { - return response; - } - const delayMs = rateLimitDelayMillis(response.headers, attempt); - yield* output.raw( - `Rate limit exceeded while ${action}. Retrying in ${rateLimitDelayText(delayMs)}.\n`, - "stderr", - ); - yield* Effect.sleep(Duration.millis(delayMs)); - } + // Repeats on a 429, reporting the exact header- or backoff-derived delay through `Schedule`'s + // own metadata. + const schedule = Schedule.fromStepWithMetadata( + Effect.succeed((meta: Schedule.InputMetadata>) => { + const previousAttempts = meta.attempt - 1; + if (meta.input.status !== 429 || previousAttempts >= DEPLOY_RATE_LIMIT_MAX_RETRIES) { + return Cause.done(meta.input); + } + const delayMs = rateLimitDelayMillis(meta.input.headers, previousAttempts, meta.now); + return output + .raw( + `Rate limit exceeded while ${action}. Retrying in ${rateLimitDelayText(delayMs)}.\n`, + "stderr", + ) + .pipe( + Effect.as([meta.input, Duration.millis(delayMs)] as [ + RateLimitableResponse, + Duration.Duration, + ]), + ); + }), + ); + return yield* request().pipe(Effect.repeat({ schedule })); }); const uploadFunctionSource = Effect.fn("functions.deploy.uploadFunctionSource")(function* ( @@ -1686,15 +1838,10 @@ const uploadFunctionSource = Effect.fn("functions.deploy.uploadFunctionSource")( bundleOnly: boolean, ) { const output = yield* Output; - const files = yield* Effect.tryPromise({ - try: async () => { - const form = await writeSourceDeployForm(sourceRoot, workdir, config, metadata, (text) => - output.raw(text, "stderr"), - ); - return form.getAll("file").flatMap((part) => (part instanceof Blob ? [part] : [])); - }, - catch: (error) => (error instanceof Error ? error : new Error(String(error))), - }); + const fs = yield* FileSystem.FileSystem; + const files = yield* writeSourceDeployForm(fs, sourceRoot, workdir, config, (text) => + output.raw(text, "stderr"), + ); const response = yield* rateLimitedRequest(`deploying function ${config.slug}`, () => api .executeRaw(operationDefinitions.v1DeployAFunction, { @@ -1721,17 +1868,16 @@ const uploadFunctionSource = Effect.fn("functions.deploy.uploadFunctionSource")( ); const body = yield* response.body; if (response.status !== 201) { - return yield* Effect.fail( - new FunctionsApiStatusError({ - status: response.status, - message: `unexpected deploy status ${response.status}: ${formatUnexpectedStatusBody(body)}`, - }), - ); + return yield* new FunctionsApiStatusError({ + status: response.status, + message: `unexpected deploy status ${response.status}: ${formatUnexpectedStatusBody(body)}`, + }); } // A 201 whose body is not the expected JSON is an API-response problem, not a // transport failure — surface it via FunctionsApiStatusError so it classifies // as api_status rather than network. return yield* Effect.try({ + // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- Schema.fromJsonString discards the native SyntaxError on malformed JSON (see decodeJsoncBytes); this raw JSON.parse keeps that message for the decode-error branch below try: () => decodeDeployFunctionResponse(JSON.parse(body)), catch: (error) => new FunctionsApiStatusError({ @@ -1763,9 +1909,8 @@ const bulkUpdateRemoteFunctions = Effect.fn("functions.deploy.bulkUpdateFunction projectRef: string, functions: ReadonlyArray, ) { - let lastError: Error | FunctionsApiStatusError | undefined; - for (let attempt = 0; attempt <= 3; attempt += 1) { - const result = yield* rateLimitedRequest("bulk updating functions", () => + const attempt = Effect.gen(function* () { + const response = yield* rateLimitedRequest("bulk updating functions", () => api .executeRaw(operationDefinitions.v1BulkUpdateFunctions, { ref: projectRef, @@ -1781,37 +1926,23 @@ const bulkUpdateRemoteFunctions = Effect.fn("functions.deploy.bulkUpdateFunction })), Effect.mapError((error) => mapTransportError("failed to bulk update", error)), ), - ).pipe( - Effect.map((response) => ({ success: true as const, response })), - Effect.catch((error) => - Effect.succeed({ - success: false as const, - error, - }), - ), ); - - if (result.success) { - const body = yield* result.response.body; - if (result.response.status === 200) { - return; - } - lastError = new FunctionsApiStatusError({ - status: result.response.status, - message: `unexpected bulk update status ${result.response.status}: ${body}`, - }); - if (result.response.status < 500) { - return yield* Effect.fail(lastError); - } - } else { - lastError = result.error; + if (response.status === 200) { + return; } + const body = yield* response.body; + return yield* new FunctionsApiStatusError({ + status: response.status, + message: `unexpected bulk update status ${response.status}: ${body}`, + }); + }); - if (attempt < 3) { - yield* Effect.sleep(Duration.millis(1_000 * 2 ** attempt)); - } - } - return yield* Effect.fail(lastError ?? new Error("failed to bulk update")); + yield* attempt.pipe( + Effect.retry({ + schedule: transientApiRetrySchedule, + while: (error) => !(error instanceof FunctionsApiStatusError) || error.status >= 500, + }), + ); }); const upsertBundledFunction = Effect.fn("functions.deploy.upsertFunction")(function* ( @@ -1820,10 +1951,10 @@ const upsertBundledFunction = Effect.fn("functions.deploy.upsertFunction")(funct bundled: BundledFunction, exists: boolean, ) { - let shouldUpdate = exists; - let lastError: Error | FunctionsApiStatusError | undefined; + const shouldUpdateRef = yield* Ref.make(exists); - for (let attempt = 0; attempt <= 3; attempt += 1) { + const attempt = Effect.gen(function* () { + const shouldUpdate = yield* Ref.get(shouldUpdateRef); const action = shouldUpdate ? "update" : "create"; const updateInput = { ref: projectRef, @@ -1849,52 +1980,50 @@ const upsertBundledFunction = Effect.fn("functions.deploy.upsertFunction")(funct }) : api.executeRaw(operationDefinitions.v1CreateAFunction, createInput); const response = yield* request.pipe( - Effect.map((value) => ({ success: true as const, value })), - Effect.catch((error) => - Effect.succeed({ - success: false as const, - error: mapTransportError(`failed to ${action} function`, error), - }), - ), + Effect.mapError((error) => mapTransportError(`failed to ${action} function`, error)), ); - if (response.success) { - const expectedStatus = shouldUpdate ? 200 : 201; - if (response.value.status === expectedStatus) { - // A success status with a malformed / unexpected JSON body is an - // API-response problem, not a transport failure — surface it via - // FunctionsApiStatusError so it classifies as api_status not network. - const body = yield* response.value.text.pipe(Effect.orElseSucceed(() => "")); - return yield* Effect.try({ - try: () => decodeDeployFunctionResponse(JSON.parse(body)), - catch: (error) => - new FunctionsApiStatusError({ - status: response.value.status, - message: `failed to read function response: ${error instanceof Error ? error.message : String(error)}`, - decode: true, - }), - }); - } - - const body = yield* response.value.text.pipe(Effect.orElseSucceed(() => "")); - if (!shouldUpdate && body.includes("Duplicated function slug")) { - shouldUpdate = true; - } - lastError = new FunctionsApiStatusError({ - status: response.value.status, - message: `unexpected ${action} function status ${response.value.status}: ${body}`, - notFoundIsInvalidInput: shouldUpdate, + const expectedStatus = shouldUpdate ? 200 : 201; + if (response.status === expectedStatus) { + // A success status with a malformed / unexpected JSON body is an + // API-response problem, not a transport failure — surface it via + // FunctionsApiStatusError so it classifies as api_status not network. + const body = yield* response.text.pipe(Effect.orElseSucceed(() => "")); + return yield* Effect.try({ + // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- Schema.fromJsonString discards the native SyntaxError on malformed JSON (see decodeJsoncBytes); this raw JSON.parse keeps that message for the decode-error branch below + try: () => decodeDeployFunctionResponse(JSON.parse(body)), + catch: (error) => + new FunctionsApiStatusError({ + status: response.status, + message: `failed to read function response: ${error instanceof Error ? error.message : String(error)}`, + decode: true, + }), }); - } else { - lastError = response.error; } - if (attempt < 3) { - yield* Effect.sleep(Duration.millis(500 * 2 ** attempt)); + const body = yield* response.text.pipe(Effect.orElseSucceed(() => "")); + let nextShouldUpdate = shouldUpdate; + if (!shouldUpdate && body.includes("Duplicated function slug")) { + nextShouldUpdate = true; + yield* Ref.set(shouldUpdateRef, true); } - } + return yield* new FunctionsApiStatusError({ + status: response.status, + message: `unexpected ${action} function status ${response.status}: ${body}`, + notFoundIsInvalidInput: nextShouldUpdate, + }); + }); - return yield* Effect.fail(lastError ?? new Error("failed to upsert function")); + // Every failure here is retried up to the bound, except a decode failure on an + // already-successful status: an unexpected status can still flip `shouldUpdateRef` (a + // `Duplicated function slug` 4xx) and succeed on the retried request, but retrying a malformed + // 200/201 body would only repeat the same write. + return yield* attempt.pipe( + Effect.retry({ + schedule: upsertRetrySchedule, + while: (error) => !(error instanceof FunctionsApiStatusError) || error.decode !== true, + }), + ); }); const deleteRemoteFunction = Effect.fn("functions.deploy.deleteFunction")(function* ( @@ -1913,44 +2042,38 @@ const deleteRemoteFunction = Effect.fn("functions.deploy.deleteFunction")(functi return; } const body = yield* response.text.pipe(Effect.orElseSucceed(() => "")); - return yield* Effect.fail( - new FunctionsApiStatusError({ - status: response.status, - message: `unexpected delete function status ${response.status}: ${body}`, - }), - ); + return yield* new FunctionsApiStatusError({ + status: response.status, + message: `unexpected delete function status ${response.status}: ${body}`, + }); }); export const discoverFunctionSlugs = Effect.fn("functions.deploy.discoverSlugs")(function* ( projectRoot: string, configDeclaredFunctions: Readonly>, ) { + const fs = yield* FileSystem.FileSystem; const functionsDir = join(projectRoot, SUPABASE_FUNCTIONS_DIR); const slugs: string[] = []; - const entries = yield* Effect.tryPromise({ - try: () => readdir(functionsDir, { withFileTypes: true }), - catch: (cause) => (cause instanceof Error ? cause : new Error(String(cause))), - }).pipe( - Effect.catch((error) => { - return "code" in error && error.code === "ENOENT" - ? Effect.succeed(undefined) - : Effect.fail(error); - }), + const entries = yield* fs.readDirectory(functionsDir).pipe( + Effect.map(Option.some), + Effect.catchTag("PlatformError", (error) => + error.reason._tag === "NotFound" + ? Effect.succeed(Option.none>()) + : Effect.fail(nativePlatformFailure(error, functionsDir)), + ), ); - if (entries !== undefined) { - for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { - if (!entry.isDirectory() && !entry.isSymbolicLink()) { - continue; - } - const slug = entry.name; + if (Option.isSome(entries)) { + // The platform `FileSystem` has no Dirent-returning `readdir`, so there's no cheap + // `isDirectory() || isSymbolicLink()` filter to check first; the entrypoint check below + // already requires `/index.ts` to exist, which rules out a non-directory entry the + // same way a separate directory check would. + for (const slug of [...entries.value].sort((left, right) => left.localeCompare(right))) { if (validateFunctionSlugMessage(slug) !== undefined) { continue; } - const hasDefaultEntrypoint = yield* Effect.promise(() => - isFile(defaultFunctionEntrypoint(functionsDir, slug)), - ); - if (hasDefaultEntrypoint) { + if (yield* isFile(fs, defaultFunctionEntrypoint(functionsDir, slug))) { slugs.push(slug); } } @@ -1983,13 +2106,14 @@ export const resolveFunctionConfigs = Effect.fn("functions.deploy.resolveConfigs readonly noVerifyJwtOverride: Option.Option; }) { const output = yield* Output; + const fs = yield* FileSystem.FileSystem; const functionsDir = join(input.projectRoot, SUPABASE_FUNCTIONS_DIR); const seenDeprecatedImportMap = new Set(); const seenFallbackImportMap = new Set(); const resolved: ResolvedDeployFunctionConfig[] = []; const fallbackImportMapPath = join(functionsDir, "import_map.json"); - const fallbackExists = yield* Effect.promise(() => isFile(fallbackImportMapPath)); + const fallbackExists = yield* isFile(fs, fallbackImportMapPath); const importMapOverride = Option.match(input.importMapOverride, { onNone: () => "", @@ -2044,11 +2168,11 @@ export const resolveFunctionConfigs = Effect.fn("functions.deploy.resolveConfigs const denoJsonc = join(functionDir, "deno.jsonc"); const deprecatedImportMap = join(functionDir, "import_map.json"); - if (yield* Effect.promise(() => isFile(denoJson))) { + if (yield* isFile(fs, denoJson)) { importMap = denoJson; - } else if (yield* Effect.promise(() => isFile(denoJsonc))) { + } else if (yield* isFile(fs, denoJsonc)) { importMap = denoJsonc; - } else if (yield* Effect.promise(() => isFile(deprecatedImportMap))) { + } else if (yield* isFile(fs, deprecatedImportMap)) { importMap = deprecatedImportMap; seenDeprecatedImportMap.add(slug); } else if (fallbackExists) { @@ -2118,9 +2242,7 @@ const deployViaApi = Effect.fn("functions.deploy.viaApi")(function* ( } if (enabled.length === 0) { - return yield* Effect.fail( - new NoFunctionsToDeployError({ message: "All Functions are up to date." }), - ); + return yield* new NoFunctionsToDeployError({ message: "All Functions are up to date." }); } const remoteBySlug = enabled.some((config) => config.verifyJwt === undefined) @@ -2302,9 +2424,7 @@ const pruneFunctions = Effect.fn("functions.deploy.prune")(function* ( ].join("\n")}\n\n`; const confirmed = yield* promptYesNo(output, yes, prompt, false); if (!confirmed) { - return yield* Effect.fail( - new FunctionDeployCancelledError({ message: CONTEXT_CANCELED_MESSAGE }), - ); + return yield* new FunctionDeployCancelledError({ message: CONTEXT_CANCELED_MESSAGE }); } for (const slug of toDelete) { @@ -2313,209 +2433,201 @@ const pruneFunctions = Effect.fn("functions.deploy.prune")(function* ( } }); -export function deployFunctions( +export const deployFunctions = Effect.fn("functions.deploy")(function* < + ResolveError, + ResolveRequirements, +>( flags: FunctionsDeployFlags, dependencies: DeployFunctionsDependencies, ) { - return Effect.gen(function* () { - const output = yield* Output; - const styleIdentifier = dependencies.styleIdentifier ?? ((text: string) => text); - const styleEmphasis = dependencies.styleEmphasis ?? ((text: string) => text); - const commandPath = ["functions", "deploy"] as const; - // Presence-based (true for `--use-api=false`, not just bare `--use-api`) — used only for - // the mutual-exclusivity check below. Behavior branches (bundler routing, --jobs guard) key - // off the resolved `flags.useApi` value instead. - const explicitUseApi = hasExplicitLongFlag(dependencies.rawArgs, commandPath, "use-api"); - const explicitUseDocker = hasExplicitLongFlag(dependencies.rawArgs, commandPath, "use-docker"); - const explicitLegacyBundle = hasExplicitLongFlag( - dependencies.rawArgs, - commandPath, - "legacy-bundle", - ); - - const changedModes = [ - explicitUseApi ? "use-api" : undefined, - explicitUseDocker ? "use-docker" : undefined, - explicitLegacyBundle ? "legacy-bundle" : undefined, - ].filter((flag): flag is string => flag !== undefined); + const output = yield* Output; + const styleIdentifier = dependencies.styleIdentifier ?? ((text: string) => text); + const styleEmphasis = dependencies.styleEmphasis ?? ((text: string) => text); + const commandPath = ["functions", "deploy"] as const; + // Presence-based (true for `--use-api=false`, not just bare `--use-api`) — used only for + // the mutual-exclusivity check below. Behavior branches (bundler routing, --jobs guard) key + // off the resolved `flags.useApi` value instead. + const explicitUseApi = hasExplicitLongFlag(dependencies.rawArgs, commandPath, "use-api"); + const explicitUseDocker = hasExplicitLongFlag(dependencies.rawArgs, commandPath, "use-docker"); + const explicitLegacyBundle = hasExplicitLongFlag( + dependencies.rawArgs, + commandPath, + "legacy-bundle", + ); - if (changedModes.length > 1) { - return yield* Effect.fail( - new ConflictingFunctionDeployFlagsError({ - message: cobraMutuallyExclusiveErrorMessage(FUNCTIONS_BUNDLER_MUTEX_GROUP, changedModes), - }), - ); - } + const changedModes = [ + explicitUseApi ? "use-api" : undefined, + explicitUseDocker ? "use-docker" : undefined, + explicitLegacyBundle ? "legacy-bundle" : undefined, + ].filter((flag): flag is string => flag !== undefined); - // `--use-api=false` alone must not force the API path — it should fall through to whatever - // `--use-docker`/`--legacy-bundle` already resolved to. - const useLocalBundler = !flags.useApi && (flags.useDocker || flags.legacyBundle); - const configuredJobs = Option.getOrElse(flags.jobs, () => 1); - const jobs = configuredJobs === 0 ? 1 : configuredJobs; - // Keyed on the resolved `--use-api` value alone, not on whether local bundling - // (Docker/legacy-bundle) is in play. - if (!flags.useApi && jobs > 1) { - return yield* Effect.fail(new Error("--jobs must be used together with --use-api")); - } + if (changedModes.length > 1) { + return yield* new ConflictingFunctionDeployFlagsError({ + message: cobraMutuallyExclusiveErrorMessage(FUNCTIONS_BUNDLER_MUTEX_GROUP, changedModes), + }); + } - const projectRef = yield* dependencies.resolveProjectRef(flags.projectRef); - // `@supabase/config` merges the matching `[remotes.*]` block over the base config, so this - // already reflects any remote function/edge_runtime overrides, through the same - // `Config.Validate`/dotenv/env-override pipeline `start`/`stop`/`status` use (see - // `functions-config.ts`). Must precede the slug-validation loop below, so an invalid - // `config.toml` is reported ahead of a malformed slug when both are wrong. - const context = yield* loadFunctionsCliConfig({ - projectRoot: dependencies.projectRoot, - projectRef, - goConfigCompat: dependencies.goConfigCompat, + // `--use-api=false` alone must not force the API path — it should fall through to whatever + // `--use-docker`/`--legacy-bundle` already resolved to. + const useLocalBundler = !flags.useApi && (flags.useDocker || flags.legacyBundle); + const configuredJobs = Option.getOrElse(flags.jobs, () => 1); + const jobs = configuredJobs === 0 ? 1 : configuredJobs; + // Keyed on the resolved `--use-api` value alone, not on whether local bundling + // (Docker/legacy-bundle) is in play. + if (!flags.useApi && jobs > 1) { + return yield* new FunctionDeployJobsRequiresApiError({ + message: "--jobs must be used together with --use-api", }); + } - if (flags.functionNames.length > 0) { - for (const slug of flags.functionNames) { - yield* validateDeploySlug(slug); - } - } + const projectRef = yield* dependencies.resolveProjectRef(flags.projectRef); + // `@supabase/config` merges the matching `[remotes.*]` block over the base config, so this + // already reflects any remote function/edge_runtime overrides, through the same + // `Config.Validate`/dotenv/env-override pipeline `start`/`stop`/`status` use (see + // `functions-config.ts`). Must precede the slug-validation loop below, so an invalid + // `config.toml` is reported ahead of a malformed slug when both are wrong. + const context = yield* loadFunctionsCliConfig({ + projectRoot: dependencies.projectRoot, + projectRef, + goConfigCompat: dependencies.goConfigCompat, + }); - const noVerifyJwtOverride = explicitBooleanFlag( - dependencies.rawArgs, - ["functions", "deploy"], - "no-verify-jwt", - flags.noVerifyJwt, - ); - // `--debug=false` must resolve to `false` — a plain presence check would get that backwards - // (same rule as `download.ts`'s own `--debug` read). - const debugEnabled = explicitBooleanLongFlag(dependencies.rawArgs, "debug") ?? false; - const deployConfig = context.loaded?.config; - const edgeRuntimeVersion = yield* resolveEdgeRuntimeVersion( - context.denoVersion, - dependencies.edgeRuntimeVersion, - ); - const configFunctions = yield* inferFunctionsManifest({ - cwd: dependencies.projectRoot, - config: deployConfig, - // Matches `loadFunctionsCliConfig`'s own options above: no ancestor directory is searched - // past `dependencies.projectRoot` for either load, so they can never resolve two - // different projects. - search: dependencies.goConfigCompat === undefined, - }); - const configDeclaredFunctions = deployConfig?.functions ?? {}; - const rawConfigFunctions = rawFunctionConfigRecord(context.loaded?.document); - yield* validateConfigFunctionSlugs(configDeclaredFunctions); - const slugs = - flags.functionNames.length > 0 - ? [...flags.functionNames] - : yield* discoverFunctionSlugs(dependencies.projectRoot, configDeclaredFunctions); - - if (slugs.length === 0) { - return yield* Effect.fail( - new NoFunctionsToDeployError({ - // Styling is text-mode only: in `--output-format json`/`stream-json` this message - // lands in the structured error payload, which must stay free of ANSI escapes. - message: `No Functions specified or found in ${ - output.format === "text" - ? styleEmphasis(SUPABASE_FUNCTIONS_DIR) - : SUPABASE_FUNCTIONS_DIR - }`, - }), - ); + if (flags.functionNames.length > 0) { + for (const slug of flags.functionNames) { + yield* validateDeploySlug(slug); } + } - const uniqueSlugs = [...new Set(slugs)]; - const configs = yield* resolveFunctionConfigs({ - slugs: uniqueSlugs, - cwd: dependencies.flagCwd, - projectRoot: dependencies.projectRoot, - supabaseDir: dependencies.supabaseDir, - configFunctions, - configDeclaredFunctions, - rawConfigFunctions, - importMapOverride: flags.importMap, - noVerifyJwtOverride, + const noVerifyJwtOverride = explicitBooleanFlag( + dependencies.rawArgs, + ["functions", "deploy"], + "no-verify-jwt", + flags.noVerifyJwt, + ); + // `--debug=false` must resolve to `false` — a plain presence check would get that backwards + // (same rule as `download.ts`'s own `--debug` read). + const debugEnabled = explicitBooleanLongFlag(dependencies.rawArgs, "debug") ?? false; + const deployConfig = context.loaded?.config; + const edgeRuntimeVersion = yield* resolveEdgeRuntimeVersion( + context.denoVersion, + dependencies.edgeRuntimeVersion, + ); + const configFunctions = yield* inferFunctionsManifest({ + cwd: dependencies.projectRoot, + config: deployConfig, + // Matches `loadFunctionsCliConfig`'s own options above: no ancestor directory is searched + // past `dependencies.projectRoot` for either load, so they can never resolve two + // different projects. + search: dependencies.goConfigCompat === undefined, + }); + const configDeclaredFunctions = deployConfig?.functions ?? {}; + const rawConfigFunctions = rawFunctionConfigRecord(context.loaded?.document); + yield* validateConfigFunctionSlugs(configDeclaredFunctions); + const slugs = + flags.functionNames.length > 0 + ? [...flags.functionNames] + : yield* discoverFunctionSlugs(dependencies.projectRoot, configDeclaredFunctions); + + if (slugs.length === 0) { + return yield* new NoFunctionsToDeployError({ + // Styling is text-mode only: in `--output-format json`/`stream-json` this message + // lands in the structured error payload, which must stay free of ANSI escapes. + message: `No Functions specified or found in ${ + output.format === "text" ? styleEmphasis(SUPABASE_FUNCTIONS_DIR) : SUPABASE_FUNCTIONS_DIR + }`, }); - const dashboardUrl = `${dependencies.dashboardUrl}/project/${projectRef}/functions`; + } - const deployWithApi = deployViaApi( - projectRef, - dependencies.projectRoot, - configs, - dependencies.api, - jobs, - ).pipe( - Effect.as(true), - Effect.catchIf( - (error): error is NoFunctionsToDeployError => error instanceof NoFunctionsToDeployError, - (error) => - (output.format === "text" - ? output.raw(`${error.message}\n`, "stderr") - : output.success(error.message, { - project_ref: projectRef, - functions: uniqueSlugs, - dashboard_url: dashboardUrl, - }) - ).pipe(Effect.as(false)), - ), - ); + const uniqueSlugs = [...new Set(slugs)]; + const configs = yield* resolveFunctionConfigs({ + slugs: uniqueSlugs, + cwd: dependencies.flagCwd, + projectRoot: dependencies.projectRoot, + supabaseDir: dependencies.supabaseDir, + configFunctions, + configDeclaredFunctions, + rawConfigFunctions, + importMapOverride: flags.importMap, + noVerifyJwtOverride, + }); + const dashboardUrl = `${dependencies.dashboardUrl}/project/${projectRef}/functions`; - const styleWarning = dependencies.styleWarning ?? ((text: string) => text); - const deployed = useLocalBundler - ? yield* Effect.gen(function* () { - if (!(yield* isDockerRunning())) { - yield* output.raw(`${styleWarning("WARNING:")} Docker is not running\n`, "stderr"); - return yield* deployWithApi; - } + const deployWithApi = deployViaApi( + projectRef, + dependencies.projectRoot, + configs, + dependencies.api, + jobs, + ).pipe( + Effect.as(true), + Effect.catchIf(Schema.is(NoFunctionsToDeployError), (error) => + (output.format === "text" + ? output.raw(`${error.message}\n`, "stderr") + : output.success(error.message, { + project_ref: projectRef, + functions: uniqueSlugs, + dashboard_url: dashboardUrl, + }) + ).pipe(Effect.as(false)), + ), + ); - // `lastExplicitLongFlagValue` preserves the "explicitly cleared" vs "never touched" - // distinction `resolveDockerNetworkMode` needs — see that function's own doc comment. - // `SUPABASE_NETWORK_ID` (env or project dotenv) is CLI-only, `undefined` for library - // callers. - const networkMode = resolveDockerNetworkMode({ - explicit: lastExplicitLongFlagValue(dependencies.rawArgs, [], "network-id"), - envOverride: - context.projectEnvValues === undefined - ? undefined - : viperEnvStringWithProjectFallback( - "SUPABASE_NETWORK_ID", - context.projectEnvValues, - ), - projectId: context.projectId, - }); - yield* deployViaDocker({ - projectId: context.projectId, - projectRef, - edgeRuntimeVersion, - functionsDir: join(dependencies.projectRoot, SUPABASE_FUNCTIONS_DIR), - configs, - api: dependencies.api, - networkMode, - verbose: debugEnabled, - styleEmphasis, - projectEnvValues: context.projectEnvValues, - }); - return true; - }) - : yield* deployWithApi; + const styleWarning = dependencies.styleWarning ?? ((text: string) => text); + const deployed = useLocalBundler + ? yield* Effect.gen(function* () { + if (!(yield* isDockerRunning())) { + yield* output.raw(`${styleWarning("WARNING:")} Docker is not running\n`, "stderr"); + return yield* deployWithApi; + } - if (!deployed) { - return; - } + // `lastExplicitLongFlagValue` preserves the "explicitly cleared" vs "never touched" + // distinction `resolveDockerNetworkMode` needs — see that function's own doc comment. + // `SUPABASE_NETWORK_ID` (env or project dotenv) is CLI-only, `undefined` for library + // callers. + const networkMode = resolveDockerNetworkMode({ + explicit: lastExplicitLongFlagValue(dependencies.rawArgs, [], "network-id"), + envOverride: + context.projectEnvValues === undefined + ? undefined + : viperEnvStringWithProjectFallback("SUPABASE_NETWORK_ID", context.projectEnvValues), + projectId: context.projectId, + }); + yield* deployViaDocker({ + projectId: context.projectId, + projectRef, + edgeRuntimeVersion, + functionsDir: join(dependencies.projectRoot, SUPABASE_FUNCTIONS_DIR), + configs, + api: dependencies.api, + networkMode, + verbose: debugEnabled, + styleEmphasis, + projectEnvValues: context.projectEnvValues, + }); + return true; + }) + : yield* deployWithApi; - if (output.format === "text") { - // Joins the raw `slugs` list, not the deduped set, so `functions deploy foo foo` prints - // "foo, foo". - yield* output.raw( - `Deployed Functions on project ${styleIdentifier(projectRef)}: ${slugs.join(", ")}\n`, - ); - yield* output.raw(`You can inspect your deployment in the Dashboard: ${dashboardUrl}\n`); - } else { - yield* output.success("Deployed Functions.", { - project_ref: projectRef, - functions: uniqueSlugs, - dashboard_url: dashboardUrl, - }); - } + if (!deployed) { + return; + } - if (flags.prune) { - yield* pruneFunctions(projectRef, configs, dependencies.api, dependencies.yes ?? false); - } - }).pipe(Effect.withSpan("functions.deploy")); -} + if (output.format === "text") { + // Joins the raw `slugs` list, not the deduped set, so `functions deploy foo foo` prints + // "foo, foo". + yield* output.raw( + `Deployed Functions on project ${styleIdentifier(projectRef)}: ${slugs.join(", ")}\n`, + ); + yield* output.raw(`You can inspect your deployment in the Dashboard: ${dashboardUrl}\n`); + } else { + yield* output.success("Deployed Functions.", { + project_ref: projectRef, + functions: uniqueSlugs, + dashboard_url: dashboardUrl, + }); + } + + if (flags.prune) { + yield* pruneFunctions(projectRef, configs, dependencies.api, dependencies.yes ?? false); + } +}); diff --git a/apps/cli/src/shared/functions/deploy.unit.test.ts b/apps/cli/src/shared/functions/deploy.unit.test.ts index eff5003774..b3c903b24c 100644 --- a/apps/cli/src/shared/functions/deploy.unit.test.ts +++ b/apps/cli/src/shared/functions/deploy.unit.test.ts @@ -120,9 +120,7 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: async (message) => { - warnings.push(message); - }, + onWarning: (message) => Effect.sync(() => warnings.push(message)), }); expect(binds.some((bind) => bind.hostPath === vendorIndexPath)).toBe(true); @@ -146,7 +144,7 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil let caught: unknown; try { await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: async () => {}, + onWarning: () => Effect.void, }); } catch (error) { caught = error; @@ -176,9 +174,7 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: async (message) => { - warnings.push(message); - }, + onWarning: (message) => Effect.sync(() => warnings.push(message)), }); expect(binds.some((bind) => formatDockerBind(bind).includes("extra.ts"))).toBe(false); @@ -201,9 +197,7 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: async (message) => { - warnings.push(message); - }, + onWarning: (message) => Effect.sync(() => warnings.push(message)), }); expect(binds.some((bind) => bind.hostPath === vendorIndexPath)).toBe(true); @@ -223,9 +217,7 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: async (message) => { - warnings.push(message); - }, + onWarning: (message) => Effect.sync(() => warnings.push(message)), }); const matches = warnings.filter( @@ -255,9 +247,7 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: async (message) => { - warnings.push(message); - }, + onWarning: (message) => Effect.sync(() => warnings.push(message)), }); expect(binds.some((bind) => formatDockerBind(bind).includes("index.mjs/core"))).toBe(false); @@ -298,9 +288,7 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: async (message) => { - warnings.push(message); - }, + onWarning: (message) => Effect.sync(() => warnings.push(message)), }); expect(binds.some((bind) => formatDockerBind(bind).includes("index.mjs"))).toBe(false); @@ -331,9 +319,7 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil const warnings: Array = []; const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: async (message) => { - warnings.push(message); - }, + onWarning: (message) => Effect.sync(() => warnings.push(message)), skipMissingImportMapTargets: true, }); @@ -359,9 +345,7 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: async (message) => { - warnings.push(message); - }, + onWarning: (message) => Effect.sync(() => warnings.push(message)), skipMissingImportMapTargets: true, }); @@ -420,9 +404,7 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil await writeFile(scopeDependency, 'export const util = "thing";\n'); const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: async (message) => { - warnings.push(message); - }, + onWarning: (message) => Effect.sync(() => warnings.push(message)), }); const hostPaths = binds.map((bind) => bind.hostPath); @@ -461,9 +443,7 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: async (message) => { - warnings.push(message); - }, + onWarning: (message) => Effect.sync(() => warnings.push(message)), }); const functionsBinds = binds.filter((bind) => bind.containerPath === resolve(functionsDir)); @@ -495,9 +475,7 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: async (message) => { - warnings.push(message); - }, + onWarning: (message) => Effect.sync(() => warnings.push(message)), }); expect(warnings).toEqual([]); @@ -521,9 +499,7 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: async (message) => { - warnings.push(message); - }, + onWarning: (message) => Effect.sync(() => warnings.push(message)), }); expect(binds.some((bind) => bind.hostPath === vendorIndexPath)).toBe(true); @@ -551,9 +527,7 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: async (message) => { - warnings.push(message); - }, + onWarning: (message) => Effect.sync(() => warnings.push(message)), }); expect(binds.some((bind) => bind.hostPath === modPath)).toBe(true); @@ -582,9 +556,7 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: async (message) => { - warnings.push(message); - }, + onWarning: (message) => Effect.sync(() => warnings.push(message)), skipMissingImportMapTargets: true, }); @@ -617,9 +589,7 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: async (message) => { - warnings.push(message); - }, + onWarning: (message) => Effect.sync(() => warnings.push(message)), }); expect(warnings.some((warning) => warning.includes("nested"))).toBe(false); diff --git a/apps/cli/src/shared/functions/serve.ts b/apps/cli/src/shared/functions/serve.ts index 579d61fdd7..fd4b3eeb11 100644 --- a/apps/cli/src/shared/functions/serve.ts +++ b/apps/cli/src/shared/functions/serve.ts @@ -1694,7 +1694,7 @@ export const resolveFunctionBindMounts = Effect.fn("functions.resolveFunctionBin skipMissingImportMapTargets: true, onWarning: (message) => { bindWarnings.push(message); - return Promise.resolve(); + return Effect.void; }, })) { binds.add(formatDockerBind(bind)); @@ -1787,7 +1787,7 @@ export const startEdgeRuntimeContainer = Effect.fn("functions.startEdgeRuntimeCo skipMissingImportMapTargets: true, onWarning: (message) => { bindWarnings.push(message); - return Promise.resolve(); + return Effect.void; }, })) { const key = formatDockerBind(bind); From 5e61ab4ab7dabb1c58841af8ba116b619f9aac8e Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sat, 3 Oct 2026 03:53:03 +0200 Subject: [PATCH 006/142] refactor(stack): move lease, registry and atomic publication into StackNamespace --- .github/workflows/test.yml | 22 +- .../stack/src/HostProcess.integration.test.ts | 31 +- packages/stack/src/HostProcess.ts | 10 +- .../stack/src/Network.integration.test.ts | 15 +- packages/stack/src/Network.ts | 6 +- packages/stack/src/Owner.integration.test.ts | 19 +- packages/stack/src/Owner.ts | 38 +- packages/stack/src/Ports.integration.test.ts | 10 +- packages/stack/src/Ports.ts | 25 +- packages/stack/src/Rpc.ts | 2 +- ...ost.container-shutdown.integration.test.ts | 6 +- ....container-sweep-retry.integration.test.ts | 8 +- .../stack/src/StackHost.integration.test.ts | 14 +- packages/stack/src/StackHost.ts | 33 +- packages/stack/src/StackNamespace.ts | 61 + packages/stack/src/State.integration.test.ts | 780 ------------ .../src/State.process.integration.test.ts | 208 ---- packages/stack/src/State.ts | 597 --------- packages/stack/src/Sweep.integration.test.ts | 57 +- packages/stack/src/Sweep.ts | 21 +- .../Supabase.native.integration.test.ts | 8 +- packages/stack/src/composition/Supabase.ts | 2 +- packages/stack/src/effect.integration.test.ts | 42 +- packages/stack/src/effect.ts | 38 +- ...dRunner.initialization.integration.test.ts | 2 +- packages/stack/src/host/CommandRunner.ts | 2 +- packages/stack/src/host/Credentials.ts | 2 +- packages/stack/src/index.ts | 2 +- .../src/internal/dispatch.integration.test.ts | 6 +- packages/stack/src/internal/host-process.ts | 2 +- .../stack/src/lifetime.integration.test.ts | 6 +- packages/stack/src/namespace/Capabilities.ts | 39 + packages/stack/src/namespace/Lease.ts | 302 +++++ .../stack/src/namespace/Namespace.e2e.test.ts | 306 +++++ .../namespace/Namespace.integration.test.ts | 1093 +++++++++++++++++ .../Namespace.windows.e2e.test.ts} | 21 +- packages/stack/src/namespace/Publication.ts | 58 + packages/stack/src/namespace/Registry.ts | 276 +++++ .../stack/src/namespace/drivers/FileSystem.ts | 72 ++ .../stack/src/namespace/drivers/Sqlite.ts | 73 ++ packages/stack/src/services/Initialization.ts | 2 +- .../src/services/Rest.integration.test.ts | 10 +- packages/stack/src/services/ServiceConfig.ts | 2 +- .../StorageGateway.integration.test.ts | 9 +- .../stack/src/shutdown.integration.test.ts | 6 +- .../stack/tests/compiled-dispatch-fixture.ts | 6 +- packages/stack/tests/host-process-fixture.ts | 24 +- ...k-fixture.ts => namespace-lock-fixture.ts} | 27 +- packages/stack/tests/owner-rpc.ts | 4 +- packages/stack/tests/publish-loop-fixture.ts | 40 + 50 files changed, 2639 insertions(+), 1806 deletions(-) create mode 100644 packages/stack/src/StackNamespace.ts delete mode 100644 packages/stack/src/State.integration.test.ts delete mode 100644 packages/stack/src/State.process.integration.test.ts delete mode 100644 packages/stack/src/State.ts create mode 100644 packages/stack/src/namespace/Capabilities.ts create mode 100644 packages/stack/src/namespace/Lease.ts create mode 100644 packages/stack/src/namespace/Namespace.e2e.test.ts create mode 100644 packages/stack/src/namespace/Namespace.integration.test.ts rename packages/stack/src/{State.windows.integration.test.ts => namespace/Namespace.windows.e2e.test.ts} (90%) create mode 100644 packages/stack/src/namespace/Publication.ts create mode 100644 packages/stack/src/namespace/Registry.ts create mode 100644 packages/stack/src/namespace/drivers/FileSystem.ts create mode 100644 packages/stack/src/namespace/drivers/Sqlite.ts rename packages/stack/tests/{state-lock-fixture.ts => namespace-lock-fixture.ts} (69%) create mode 100644 packages/stack/tests/publish-loop-fixture.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0caa0635d6..e2b69680d5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -123,6 +123,12 @@ jobs: - name: Run integration tests run: pnpm run test:integration + # Not covered by test:integration above (a different vitest project); runs here so the + # registry cross-process and publication-crash scenarios run on Linux too. + - name: Run stack namespace e2e tests + working-directory: packages/stack + run: pnpm test:e2e:run src/namespace/Namespace.e2e.test.ts --passWithNoTests=false + test-stack-ports: if: | !startsWith(github.head_ref, 'release-notes/') && @@ -155,16 +161,24 @@ jobs: pnpm test:integration src/Ports.integration.test.ts src/storage/DirectoryCopy.integration.test.ts - src/State.integration.test.ts - src/State.windows.integration.test.ts - src/State.process.integration.test.ts + src/namespace/Namespace.integration.test.ts src/HostProcess.integration.test.ts src/Network.integration.test.ts src/Proxy.integration.test.ts src/HttpProxy.integration.test.ts --passWithNoTests=false - # Windows-only CLI subset: paths, PATH lookup, and permission semantics diverge there. + - name: Run stack namespace e2e tests + working-directory: packages/stack + run: pnpm test:e2e:run src/namespace/Namespace.e2e.test.ts --passWithNoTests=false + + # Windows-only: sharing-violation retries and CLI paths, PATH lookup, and permission + # semantics that only diverge there. + - name: Run stack namespace Windows e2e tests + if: runner.os == 'Windows' + working-directory: packages/stack + run: pnpm test:e2e:run src/namespace/Namespace.windows.e2e.test.ts --passWithNoTests=false + - name: Run CLI integration tests if: runner.os == 'Windows' working-directory: apps/cli diff --git a/packages/stack/src/HostProcess.integration.test.ts b/packages/stack/src/HostProcess.integration.test.ts index f6a89455ba..23ce267e69 100644 --- a/packages/stack/src/HostProcess.integration.test.ts +++ b/packages/stack/src/HostProcess.integration.test.ts @@ -31,7 +31,7 @@ import { } from "./HostProcess.ts"; import { discover } from "./effect.ts"; import { watchLeaseRelease } from "../tests/owner.ts"; -import * as State from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; class ProcessTestError extends Data.TaggedError("ProcessTestError")<{ readonly message: string }> {} @@ -39,7 +39,7 @@ const fixtureEntrypoint = fileURLToPath( new URL("../tests/host-process-fixture.ts", import.meta.url), ); -const savedStack = (root: string, stackName: string): State.SavedStack => ({ +const savedStack = (root: string, stackName: string): StackNamespace.SavedStack => ({ id: "stack", runtime: "native", identity: { projectRoot: root, branchContext: "main", stackName }, @@ -77,10 +77,21 @@ const silentListener = Effect.acquireRelease( ); const makeTestState = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), ); +/** Writes a holder record directly, simulating a process that published then crashed before retracting. */ +const writeStaleHolder = (root: string, id: string, record: StackNamespace.LeaseHolder) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const encoded = yield* Schema.encodeEffect(Schema.fromJsonString(StackNamespace.LeaseHolder))( + record, + ); + yield* fs.writeFileString(path.join(root, id, "owner.json"), encoded); + }); + type ChildHandle = { readonly child: ChildProcessSpawner.ChildProcessHandle; closed: boolean; @@ -187,7 +198,11 @@ const bestEffortShutdown = (stateRoot: string) => (access: HostAccess) => }), ).pipe(Effect.exit, Effect.asVoid); -const bestEffortShutdownByState = (stateRoot: string, state: State.Interface, stackId: string) => +const bestEffortShutdownByState = ( + stateRoot: string, + state: StackNamespace.Interface, + stackId: string, +) => Effect.exit(connectHost(state, stackId).pipe(Effect.flatMap(bestEffortShutdown(stateRoot)))).pipe( Effect.asVoid, ); @@ -330,7 +345,7 @@ it.live("ignores a stale endpoint record once no process holds the lease", () => const state = yield* makeTestState(root); yield* state.save(savedStack(root, "local")); const unresponsive = yield* silentListener; - yield* state.publishHolder("stack", { + yield* writeStaleHolder(root, "stack", { role: "owner", secret: "stale", port: unresponsive.port, @@ -368,8 +383,8 @@ it.live("discovers dead stacks from their free leases without contacting recorde const ids = ["dead-a", "dead-b", "dead-c"]; for (const id of ids) { yield* state.save({ ...savedStack(root, id), id }); - yield* Effect.scoped(state.lease(id)); - yield* state.publishHolder(id, { + yield* Effect.scoped(state.acquireLease(id)); + yield* writeStaleHolder(root, id, { role: "owner", secret: "stale", port: unresponsive.port, diff --git a/packages/stack/src/HostProcess.ts b/packages/stack/src/HostProcess.ts index 88f34aa4de..c3754bedbb 100644 --- a/packages/stack/src/HostProcess.ts +++ b/packages/stack/src/HostProcess.ts @@ -31,7 +31,11 @@ import { HOST_PROCESS_DISPATCH_SENTINEL, isBunVirtualPath } from "./internal/dis import { failureMessage } from "./internal/failure-message.ts"; import { stackSourceDigest } from "./internal/release.ts"; import { StackRpc } from "./Rpc.ts"; -import { SavedStack, type Interface as StateInterface, type StateError } from "./State.ts"; +import { + SavedStack, + type Interface as StateInterface, + type NamespaceError, +} from "./StackNamespace.ts"; declare const SUPABASE_STACK_BUILD_ID: string | undefined; @@ -259,7 +263,7 @@ export const connectHost = Effect.fn("HostProcess.connectHost")(function* ( options: ConnectOptions = {}, ): Effect.fn.Return< HostAccess, - HostProcessError | StateError, + HostProcessError | NamespaceError, HttpClient.HttpClient | FileSystem.FileSystem | Path.Path | Crypto.Crypto > { const stack = yield* registered(state, stackId); @@ -553,7 +557,7 @@ export const launchHost = Effect.fn("HostProcess.launchHost")(function* ( options: LaunchOptions, ): Effect.fn.Return< HostAccess, - HostProcessError | StateError, + HostProcessError | NamespaceError, Scope.Scope | HttpClient.HttpClient | FileSystem.FileSystem | Path.Path | Crypto.Crypto > { const entrypoint = options.entrypoint ?? hostEntrypointFor(import.meta.url); diff --git a/packages/stack/src/Network.integration.test.ts b/packages/stack/src/Network.integration.test.ts index 680424c0ad..9ca7022628 100644 --- a/packages/stack/src/Network.integration.test.ts +++ b/packages/stack/src/Network.integration.test.ts @@ -6,20 +6,20 @@ import { createServer } from "node:http"; // oxlint-disable-line effecttsgo/node import { HttpClient } from "effect/unstable/http"; import * as Network from "./Network.ts"; import { DOCKER_HOST_ALIAS } from "./runtime/Container.ts"; -import * as State from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; const makeTestState = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), ); const makeTestNetwork = (options: { readonly stackId: string; readonly runtime: Network.NetworkRuntime; - readonly state: State.Interface; + readonly state: StackNamespace.Interface; }) => Layer.build( Network.layer({ stackId: options.stackId, runtime: options.runtime }).pipe( - Layer.provide(Layer.succeed(State.Service, options.state)), + Layer.provide(Layer.succeed(StackNamespace.Service, options.state)), ), ).pipe(Effect.map((context) => Context.get(context, Network.Service))); @@ -260,7 +260,10 @@ it.live("can bind a shared listener after saving its first assignment failed", ( Effect.flatMap((fail) => fail ? Effect.fail( - new State.StateError({ operation: "save", message: "write failed" }), + new StackNamespace.NamespaceError({ + operation: "save", + message: "write failed", + }), ) : state.save(value), ), diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index bb80763509..4315b18279 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -1,6 +1,6 @@ import { Context, Data, Effect, Exit, Layer, Ref, Scope, Semaphore } from "effect"; import { DOCKER_HOST_ALIAS } from "./runtime/Container.ts"; -import * as State from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; import { makePorts, PortError } from "./Ports.ts"; import { bindTcp, serveTcp, type BackendAddress, type ProxyError } from "./Proxy.ts"; import { makeHttpProxy, type HttpProxy, type HttpRoute } from "./HttpProxy.ts"; @@ -69,7 +69,7 @@ const errorFor = (operation: string, cause: unknown) => const makeNetwork = (options: { readonly stackId: string; readonly runtime: NetworkRuntime; - readonly state: State.Interface; + readonly state: StackNamespace.Interface; }) => Effect.gen(function* () { const ports = yield* makePorts(options.state).pipe( @@ -345,7 +345,7 @@ export const layer = (options: { readonly stackId: string; readonly runtime: Net Layer.effect( Service, Effect.gen(function* () { - const state = yield* State.Service; + const state = yield* StackNamespace.Service; return yield* makeNetwork({ ...options, state }); }).pipe(Effect.map(Service.of)), ); diff --git a/packages/stack/src/Owner.integration.test.ts b/packages/stack/src/Owner.integration.test.ts index 029119d629..05c8efe356 100644 --- a/packages/stack/src/Owner.integration.test.ts +++ b/packages/stack/src/Owner.integration.test.ts @@ -7,16 +7,16 @@ import { tmpdir } from "node:os"; import { RpcTest } from "effect/unstable/rpc"; import * as Owner from "./Owner.ts"; import { OwnerRpc } from "./Rpc.ts"; -import * as State from "./State.ts"; -import type { SavedStack } from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; +import type { SavedStack } from "./StackNamespace.ts"; import { DEFAULT_LOCAL_JWT_SECRET } from "./Defaults.ts"; import { ServiceCreation, type ServiceCreationInput } from "./services/Catalog.ts"; import { ownerFor } from "../tests/owner-rpc.ts"; const stateFor = (root: string) => Effect.gen(function* () { - const context = yield* Layer.build(State.layer({ root })); - return Context.get(context, State.Service); + const context = yield* Layer.build(StackNamespace.layer({ root })); + return Context.get(context, StackNamespace.Service); }); const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; @@ -142,7 +142,7 @@ it.live("forwards and rotates saved identity across composed services in one own endpoints: { http: { port: "auto" as const } }, }, ]; - const stackKeys = (suffix: string, gotrueJwtKeys: string): State.StackKeysInput => ({ + const stackKeys = (suffix: string, gotrueJwtKeys: string): StackNamespace.StackKeysInput => ({ publishableKey: `publishable-${suffix}`, secretKey: `secret-${suffix}`, anonKey: `anon-${suffix}`, @@ -333,7 +333,7 @@ it.effect("publishes service removal and composition pruning together", () => const state = yield* stateFor(`${root}/state`); yield* state.save(stack); const removalWrite = yield* Ref.make(false); - const failingState: State.Interface = { + const failingState: StackNamespace.Interface = { ...state, save: (next) => Ref.get(removalWrite).pipe( @@ -341,7 +341,10 @@ it.effect("publishes service removal and composition pruning together", () => next.instances.length === 0 ? alreadyWritten ? Effect.fail( - new State.StateError({ operation: "write", message: "injected failure" }), + new StackNamespace.NamespaceError({ + operation: "write", + message: "injected failure", + }), ) : Ref.set(removalWrite, true).pipe(Effect.andThen(state.save(next))) : state.save(next), @@ -495,7 +498,7 @@ it.effect("isolates owner graphs built in one scope", () => saved, root: `${root}/data`, cacheRoot, - }).pipe(Layer.provide(Layer.succeed(State.Service, state))), + }).pipe(Layer.provide(Layer.succeed(StackNamespace.Service, state))), memoMap, scope, ).pipe( diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index b36d9fcfa1..9271d7ea09 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -64,13 +64,13 @@ import type { CatalogError } from "./services/Recipe.ts"; import * as Container from "./runtime/Container.ts"; import { projectSegmentFor } from "./identity/Identity.ts"; import { stackError, type OwnerRpc } from "./Rpc.ts"; -import * as State from "./State.ts"; -import type { SavedStack, StackCredentials, StackKeysInput } from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; +import type { SavedStack, StackCredentials, StackKeysInput } from "./StackNamespace.ts"; import { makeDockerHelperRegistry } from "./storage/DockerHelperRegistry.ts"; export interface OwnerOptions { readonly saved: SavedStack; - readonly state: State.Interface; + readonly state: StackNamespace.Interface; readonly root: string; readonly cacheRoot: string; /** Shares one host-gateway probe with the host's other container runtimes. */ @@ -99,12 +99,15 @@ type NamespaceError = | Orchestrator.OrchestratorError | Orchestrator.LifecycleError | Network.NetworkError - | State.StateError + | StackNamespace.NamespaceError | Effect.Error>; export interface Interface { readonly handlers: Handlers; - readonly getStackCredentials: Effect.Effect; + readonly getStackCredentials: Effect.Effect< + StackCredentials, + StackNamespace.NamespaceError | CredentialError + >; readonly namespace: { /** Stops every instance after in-flight definition changes settle, then removes containers. */ readonly stop: Effect.Effect; @@ -222,17 +225,18 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { }), ); - const readSaved = options.state - .read(stackId) - .pipe( - Effect.flatMap((saved) => - saved === undefined - ? Effect.fail( - new State.StateError({ operation: "read", message: "Saved stack is missing" }), - ) - : Effect.succeed(saved), - ), - ); + const readSaved = options.state.read(stackId).pipe( + Effect.flatMap((saved) => + saved === undefined + ? Effect.fail( + new StackNamespace.NamespaceError({ + operation: "read", + message: "Saved stack is missing", + }), + ) + : Effect.succeed(saved), + ), + ); const updateState = (update: (current: SavedStack) => SavedStack) => options.state.withLock( readSaved.pipe(Effect.flatMap((current) => options.state.save(update(current)))), @@ -709,7 +713,7 @@ export const layer = (options: Omit) => Layer.effect( Service, Effect.gen(function* () { - const state = yield* State.Service; + const state = yield* StackNamespace.Service; return Service.of(yield* makeOwner({ ...options, state })); }), ).pipe( diff --git a/packages/stack/src/Ports.integration.test.ts b/packages/stack/src/Ports.integration.test.ts index e5c4f9bb0d..0ec892a955 100644 --- a/packages/stack/src/Ports.integration.test.ts +++ b/packages/stack/src/Ports.integration.test.ts @@ -2,11 +2,11 @@ import { NodeServices, NodeSocketServer } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; import { Cause, Context, Effect, Exit, FileSystem, Layer, Option, Ref, Scope } from "effect"; import { makePorts, PortError } from "./Ports.ts"; -import * as State from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; const makeTestState = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), ); const bind = (host: string, port: number) => @@ -169,10 +169,10 @@ it.live("names the last bind failure when no public port is available", () => ); const saveStack = ( - state: State.Interface, + state: StackNamespace.Interface, root: string, id: string, - ports: State.SavedStack["ports"] = [], + ports: StackNamespace.SavedStack["ports"] = [], ) => state.save({ id, diff --git a/packages/stack/src/Ports.ts b/packages/stack/src/Ports.ts index d45216416b..82c87f3e15 100644 --- a/packages/stack/src/Ports.ts +++ b/packages/stack/src/Ports.ts @@ -1,6 +1,6 @@ import { Cause, Data, Effect, Exit, Hash, Option, Scope } from "effect"; import * as Net from "node:net"; -import type * as State from "./State.ts"; +import type * as StackNamespace from "./StackNamespace.ts"; const portBase = 20000; /** Stays below the Linux ephemeral range, per the [architecture ADR](../../../docs/adr/0017-simplified-managed-stack-architecture.md). */ @@ -22,7 +22,7 @@ export interface PortRequest { } /** Spreads the scan across the span so separate checkouts, stacks, and keys start apart. */ -const scanStart = (stack: State.SavedStack, key: string) => +const scanStart = (stack: StackNamespace.SavedStack, key: string) => Math.abs(Hash.string(`${stack.identity.projectRoot}:${stack.id}:${key}`)) % portSpan; /** A wildcard listener is reachable through loopback, where a same-port loopback listener answers too. */ @@ -52,15 +52,19 @@ const loopbackOccupied = (port: number) => concurrency: "unbounded", }).pipe(Effect.map((answers) => answers.some(Boolean))); -const claimantOf = (stacks: ReadonlyArray, stackId: string, port: number) => +const claimantOf = ( + stacks: ReadonlyArray, + stackId: string, + port: number, +) => stacks.find((other) => other.id !== stackId && other.ports.some((claim) => claim.port === port)) ?.id; const resolveRequest = ( - stack: State.SavedStack, + stack: StackNamespace.SavedStack, request: PortRequest, ): Effect.Effect< - { readonly saved: State.PortClaim | undefined; readonly requested: number | "auto" }, + { readonly saved: StackNamespace.PortClaim | undefined; readonly requested: number | "auto" }, PortError > => { const saved = stack.ports.find((entry) => entry.key === request.key); @@ -89,7 +93,10 @@ const resolveRequest = ( }; /** Claims steer auto allocation away from saved stacks; live listeners and binds decide conflicts for fixed ports. */ -export const makePorts = (state: State.Interface, platform: NodeJS.Platform = process.platform) => +export const makePorts = ( + state: StackNamespace.Interface, + platform: NodeJS.Platform = process.platform, +) => Effect.sync(() => { const describe = (id: string) => state.read(id).pipe( @@ -101,7 +108,11 @@ export const makePorts = (state: State.Interface, platform: NodeJS.Platform = pr Effect.orElseSucceed(() => id), ); - const claimedBy = (stacks: ReadonlyArray, stackId: string, port: number) => { + const claimedBy = ( + stacks: ReadonlyArray, + stackId: string, + port: number, + ) => { const claimant = claimantOf(stacks, stackId, port); return claimant === undefined ? Effect.succeed("") diff --git a/packages/stack/src/Rpc.ts b/packages/stack/src/Rpc.ts index 9b296509c4..acf913c2e9 100644 --- a/packages/stack/src/Rpc.ts +++ b/packages/stack/src/Rpc.ts @@ -4,7 +4,7 @@ import { ServiceCreation, ServiceCreationInput } from "./services/Catalog.ts"; import { snapshotScopes } from "./services/DatabaseSnapshot.ts"; import { causeMessage, CompositionConfig, OrchestratorError } from "./Orchestrator.ts"; import { CommandInvocation } from "./Commands.ts"; -import { StackKeysInput } from "./State.ts"; +import { StackKeysInput } from "./StackNamespace.ts"; import { failureMessage } from "./internal/failure-message.ts"; const Outcome = Schema.Struct({ diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index 4c0971f97c..a89db6e76e 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -2,7 +2,7 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; import { Context, Crypto, Effect, FileSystem, Layer, Path, Redacted, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import * as State from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; import { hasReason, launchHost, @@ -40,8 +40,8 @@ const docker = Effect.fn("StackHostContainerShutdownTest.docker")((args: Readonl ); const stateFor = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), ); const containers = (stackId: string, dataRoot: string) => diff --git a/packages/stack/src/StackHost.container-sweep-retry.integration.test.ts b/packages/stack/src/StackHost.container-sweep-retry.integration.test.ts index 5e893aaa75..e652e857ed 100644 --- a/packages/stack/src/StackHost.container-sweep-retry.integration.test.ts +++ b/packages/stack/src/StackHost.container-sweep-retry.integration.test.ts @@ -2,15 +2,15 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; import { Context, Deferred, Effect, FileSystem, Layer, Sink, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import * as State from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; import * as Owner from "./Owner.ts"; import * as CommandRunner from "./host/CommandRunner.ts"; import { bindControl, makeRuntime } from "./StackHost.ts"; import { shutdownOwner } from "../tests/owner.ts"; const stateFor = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), ); it.live("retains saved state when destroy sweep fails and retries after engine recovery", () => @@ -63,7 +63,7 @@ it.live("retains saved state when destroy sweep fails and retries after engine r }).pipe( Layer.provide( Layer.merge( - Layer.succeed(State.Service, state), + Layer.succeed(StackNamespace.Service, state), Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, engine), ), ), diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts index 914f877ed2..165ff7afb8 100644 --- a/packages/stack/src/StackHost.integration.test.ts +++ b/packages/stack/src/StackHost.integration.test.ts @@ -31,7 +31,7 @@ import { import * as Owner from "./Owner.ts"; import { OrchestratorError } from "./Orchestrator.ts"; import { CommandEvent, StackError } from "./Rpc.ts"; -import * as State from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; import { bindControl, makeRuntime } from "./StackHost.ts"; import { shutdownOwner } from "../tests/owner.ts"; import { postgres } from "./Commands.ts"; @@ -41,8 +41,8 @@ class HostTestError extends Data.TaggedError("HostTestError")<{ readonly message const stateFor = (root: string) => Effect.gen(function* () { - const context = yield* Layer.build(State.layer({ root })); - return Context.get(context, State.Service); + const context = yield* Layer.build(StackNamespace.layer({ root })); + return Context.get(context, StackNamespace.Service); }); const hostTestError = (cause: unknown) => @@ -50,14 +50,14 @@ const hostTestError = (cause: unknown) => const ownerFor = (options: { readonly saved: Parameters[0]["saved"]; - readonly state: State.Interface; + readonly state: StackNamespace.Interface; readonly root: string; readonly cacheRoot: string; }) => { const { state, ...layerOptions } = options; return Effect.gen(function* () { const context = yield* Layer.build( - Owner.layer(layerOptions).pipe(Layer.provide(Layer.succeed(State.Service, state))), + Owner.layer(layerOptions).pipe(Layer.provide(Layer.succeed(StackNamespace.Service, state))), ); return Context.get(context, Owner.Service); }); @@ -135,7 +135,7 @@ const awaitClosed = (socket: Net.Socket) => const inProcessRuntime = ( owner: Parameters[0], - state: State.Interface, + state: StackNamespace.Interface, root: string, ) => Effect.gen(function* () { @@ -812,7 +812,7 @@ const disconnectFixture = (prefix: string) => const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix }); const state = yield* stateFor(`${root}/state`); - const saved: State.SavedStack = { + const saved: StackNamespace.SavedStack = { id: "stack", runtime: "native", identity: { projectRoot: root, branchContext: "main", stackName: prefix }, diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index e2571409c3..5e3168ba41 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -42,7 +42,7 @@ import { projectSegmentFor } from "./identity/Identity.ts"; import * as Owner from "./Owner.ts"; import { StackError, stackError, StackRpc, type RunCommandPayload } from "./Rpc.ts"; import { makeHostGateway } from "./runtime/Container.ts"; -import * as State from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; import { sweepOrphans } from "./Sweep.ts"; import { makeCommandAttachments } from "./host/CommandAttachments.ts"; import * as CommandRunner from "./host/CommandRunner.ts"; @@ -52,7 +52,7 @@ export interface StackHostOptions { readonly cacheRoot: string; readonly stackId: string; /** Registers this definition once the owner holds the lease; the stack must not exist. */ - readonly register?: State.SavedStack; + readonly register?: StackNamespace.SavedStack; readonly release?: string; readonly onReady?: (access: HostAccess) => Effect.Effect; } @@ -393,20 +393,25 @@ export const runStackHost = Effect.fn("StackHost.run")( ), ), ); - const stateContext = yield* Layer.build(State.layer({ root: options.stateRoot })); - const state = Context.get(stateContext, State.Service); + const stateContext = yield* Layer.build(StackNamespace.layer({ root: options.stateRoot })); + const state = Context.get(stateContext, StackNamespace.Service); const id = options.stackId; // The lease is released last, after every owned process and listener has closed. - if (!(yield* state.lease(id))) - return yield* new StackHostError({ - operation: "lease", - message: `Another owner holds the lease of stack ${id}`, - reason: "lease-held", - }); + const lease = yield* state.acquireLease(id).pipe( + Effect.catchTag( + "Namespace.LeaseHeldError", + () => + new StackHostError({ + operation: "lease", + message: `Another owner holds the lease of stack ${id}`, + reason: "lease-held", + }), + ), + ); const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; yield* fs.truncate(state.ownerLog(id)).pipe(Effect.ignore); - yield* state.retractHolder(id); + yield* lease.retractHolder; const register = options.register; if (register !== undefined) yield* state.withLock( @@ -454,7 +459,7 @@ export const runStackHost = Effect.fn("StackHost.run")( runtime: saved.runtime, hostGateway, }), - ).pipe(Layer.provide(Layer.succeed(State.Service, state))), + ).pipe(Layer.provide(Layer.succeed(StackNamespace.Service, state))), ); const owner = Context.get(services, Owner.Service); const endpoint: HostEndpoint = { @@ -481,8 +486,8 @@ export const runStackHost = Effect.fn("StackHost.run")( ), ); yield* runtime.serve; - yield* Effect.addFinalizer(() => state.retractHolder(id).pipe(Effect.ignore)); - yield* state.publishHolder(id, { + yield* Effect.addFinalizer(() => lease.retractHolder.pipe(Effect.ignore)); + yield* lease.publishHolder({ role: "owner", secret, port: endpoint.port, diff --git a/packages/stack/src/StackNamespace.ts b/packages/stack/src/StackNamespace.ts new file mode 100644 index 0000000000..ec892dc158 --- /dev/null +++ b/packages/stack/src/StackNamespace.ts @@ -0,0 +1,61 @@ +import { Context, Effect, FileSystem, Layer, Path } from "effect"; +import { namespaceError, type NamespaceError } from "./namespace/Capabilities.ts"; +import * as FsDriver from "./namespace/drivers/FileSystem.ts"; +import * as Lease from "./namespace/Lease.ts"; +import * as Registry from "./namespace/Registry.ts"; +import { restrictDirectoryToOwner } from "./runtime/postgres-user.ts"; + +export { NamespaceError } from "./namespace/Capabilities.ts"; +export { LeaseHeldError, LeaseHolder } from "./namespace/Lease.ts"; +export { + PortClaim, + SavedStack, + StackCredentials, + StackKeysInput, + StackLifetime, + type StackClaims, +} from "./namespace/Registry.ts"; + +export interface Interface extends Registry.Interface, Lease.Interface {} + +export class Service extends Context.Service()( + "@supabase/stack/StackNamespace", +) {} + +export interface Options { + readonly root: string; + readonly platform?: NodeJS.Platform; + readonly onInvalidState?: (id: string, error: NamespaceError) => Effect.Effect; + /** Observes a lease request that found the lease held and is waiting for it. */ + readonly onLeaseContended?: (id: string) => Effect.Effect; +} + +const make = Effect.fn("Namespace.acquire")(function* ( + options: Options, +): Effect.fn.Return { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = path.normalize(options.root); + yield* fs + .makeDirectory(root, { recursive: true }) + .pipe(Effect.mapError((cause) => namespaceError("root", cause))); + yield* restrictDirectoryToOwner(fs, root).pipe( + Effect.mapError((cause) => namespaceError("root", cause)), + ); + yield* FsDriver.assertHardLinkSupport(fs, path, root); + const registry = yield* Registry.make({ + root, + platform: options.platform, + onInvalidState: options.onInvalidState, + }); + const lease = yield* Lease.make({ + root, + platform: options.platform, + onLeaseContended: options.onLeaseContended, + isRegistered: (id) => registry.read(id).pipe(Effect.map((saved) => saved !== undefined)), + }); + return { ...registry, ...lease }; +}); + +export const layer = (options: Options) => + Layer.effect(Service, make(options).pipe(Effect.map(Service.of))); diff --git a/packages/stack/src/State.integration.test.ts b/packages/stack/src/State.integration.test.ts deleted file mode 100644 index 19a1548de3..0000000000 --- a/packages/stack/src/State.integration.test.ts +++ /dev/null @@ -1,780 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { TestClock } from "effect/testing"; -import { - Context, - DateTime, - Deferred, - Effect, - Exit, - Fiber, - FileSystem, - Layer, - Path, - PlatformError, - Ref, - Schema, - Scope, -} from "effect"; -import * as State from "./State.ts"; -import type { SavedStack } from "./State.ts"; - -const makeTestState = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), - ); - -const initial: SavedStack = { - id: "stack-main", - identity: { - projectRoot: "/tmp/project", - branchContext: "main", - stackName: "local", - }, - runtime: "docker", - instances: [], - lifetime: "detached", - composition: { members: [], dependencies: [] }, - ports: [], -}; - -const run = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - -describe("durable stack state", () => { - it.live("reopens saved state and serializes concurrent read-modify-write operations", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-" }); - const first = yield* makeTestState(root); - const second = yield* makeTestState(root); - yield* first.save(initial); - - const add = (store: typeof first, id: string) => - store.withLock( - Effect.gen(function* () { - const current = yield* store.read(initial.id); - if (current === undefined) return yield* Effect.die("state disappeared"); - yield* store.save({ - ...current, - instances: [ - ...current.instances, - { id, creation: { service: "mail", config: {} } }, - ], - }); - }), - ); - yield* Effect.all([add(first, "db-one"), add(second, "db-two")], { - concurrency: "unbounded", - }); - - const reopened = yield* (yield* makeTestState(root)).read(initial.id); - expect(reopened?.instances.map((instance) => instance.id).sort()).toEqual([ - "db-one", - "db-two", - ]); - expect(yield* fs.stat(path.join(root, "stack-main", "state.json"))).toBeDefined(); - }), - ), - ); - - it.effect("rejects unsafe paths and malformed documents without changing valid state", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-invalid-" }); - const store = yield* makeTestState(root); - yield* store.save(initial); - const unsafe = yield* store.read("../outside").pipe(Effect.exit); - expect(Exit.isFailure(unsafe)).toBe(true); - - yield* fs.makeDirectory(`${root}/broken`); - yield* fs.writeFileString(`${root}/broken/state.json`, '{"id":42}'); - const malformed = yield* store.read("broken").pipe(Effect.exit); - expect(Exit.isFailure(malformed)).toBe(true); - if (Exit.isFailure(malformed)) { - expect(String(malformed.cause)).toContain("stack broken"); - expect(String(malformed.cause)).toContain("state.json"); - } - const valid = yield* store.read(initial.id); - expect(Schema.is(State.SavedStack)(valid)).toBe(true); - }), - ), - ); - - it.effect("reports replacement errno and destination while preserving the prior state", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-state-rename-", - }); - const renameCalls = yield* Ref.make(0); - const failRename = yield* Ref.make(false); - const firstFailure = yield* Deferred.make(); - const injectedFs = Layer.effect( - FileSystem.FileSystem, - Effect.succeed({ - ...fs, - rename: (from: string, to: string) => - Ref.updateAndGet(renameCalls, (calls) => calls + 1).pipe( - Effect.flatMap(() => - Effect.map(Ref.get(failRename), (shouldFail) => ({ - shouldFail, - })), - ), - Effect.flatMap(({ shouldFail }) => - shouldFail && from.endsWith("state.json") && to.endsWith("state.json") - ? Deferred.succeed(firstFailure, undefined).pipe( - Effect.andThen( - Effect.fail( - PlatformError.systemError({ - _tag: "Unknown", - module: "FileSystem", - method: "rename", - pathOrDescriptor: to, - description: "injected replacement failure", - cause: Object.assign(new Error("sharing violation"), { - code: "EPERM", - }), - }), - ), - ), - ) - : fs.rename(from, to), - ), - ), - }), - ); - const store = yield* Layer.build( - State.layer({ root, platform: "win32" }).pipe(Layer.provide(injectedFs)), - ).pipe(Effect.map((context) => Context.get(context, State.Service))); - yield* store.save(initial); - yield* Ref.set(renameCalls, 0); - yield* Ref.set(failRename, true); - const saving = yield* store - .save({ - ...initial, - identity: { ...initial.identity, stackName: "next" }, - }) - .pipe(Effect.forkScoped); - yield* Deferred.await(firstFailure); - yield* TestClock.adjust("950 millis"); - const error = yield* Fiber.join(saving).pipe(Effect.flip); - expect(error.operation).toBe("publish"); - expect(error.message).toContain("EPERM"); - expect(error.message).toContain(path.join(root, "stack-main", "state.json")); - expect(yield* Ref.get(renameCalls)).toBe(13); - expect(yield* store.read(initial.id)).toEqual(initial); - expect( - (yield* fs.readDirectory(root)).some((entry) => entry.startsWith(".state-write-")), - ).toBe(false); - }), - ), - ); - - it.effect("recovers from each Windows replacement errno using the same temporary state", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - for (const code of ["EPERM", "EACCES", "EBUSY"]) { - const root = yield* fs.makeTempDirectoryScoped({ - prefix: `stack-state-retry-${code}-`, - }); - const calls = yield* Ref.make(0); - const temporaryPaths = yield* Ref.make>([]); - const failOnce = yield* Ref.make(false); - const firstFailure = yield* Deferred.make(); - const injectedFs = Layer.effect( - FileSystem.FileSystem, - Effect.succeed({ - ...fs, - rename: (from: string, to: string) => - Effect.gen(function* () { - yield* Ref.update(calls, (count) => count + 1); - if (yield* Ref.get(failOnce)) { - yield* Ref.set(failOnce, false); - yield* Ref.update(temporaryPaths, (paths) => [...paths, from]); - yield* Deferred.succeed(firstFailure, undefined); - return yield* PlatformError.systemError({ - _tag: - code === "EACCES" - ? "PermissionDenied" - : code === "EBUSY" - ? "Busy" - : "Unknown", - module: "FileSystem", - method: "rename", - pathOrDescriptor: to, - cause: Object.assign(new Error("sharing violation"), { - code, - }), - }); - } - yield* Ref.update(temporaryPaths, (paths) => [...paths, from]); - return yield* fs.rename(from, to); - }), - }), - ); - const store = yield* Layer.build( - State.layer({ root, platform: "win32" }).pipe(Layer.provide(injectedFs)), - ).pipe(Effect.map((context) => Context.get(context, State.Service))); - yield* store.save(initial); - yield* Ref.set(calls, 0); - yield* Ref.set(temporaryPaths, []); - yield* Ref.set(failOnce, true); - const publishing = yield* store - .save({ - ...initial, - identity: { ...initial.identity, stackName: code }, - }) - .pipe(Effect.forkScoped); - yield* Deferred.await(firstFailure); - expect(yield* store.read(initial.id)).toEqual(initial); - expect( - (yield* fs.readDirectory(root)).some((entry) => entry.startsWith(".state-write-")), - ).toBe(true); - yield* TestClock.adjust("10 millis"); - yield* Fiber.join(publishing); - expect(yield* Ref.get(calls)).toBe(2); - const retriedPaths = yield* Ref.get(temporaryPaths); - expect(retriedPaths).toHaveLength(2); - expect(retriedPaths[0]).toContain(".state-write-"); - expect(retriedPaths[1]).toBe(retriedPaths[0]); - expect((yield* store.read(initial.id))?.identity.stackName).toBe(code); - expect( - (yield* fs.readDirectory(root)).some((entry) => entry.startsWith(".state-write-")), - ).toBe(false); - } - }), - ), - ); - - it.effect("does not retry non-Windows or unrelated replacement errors", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - for (const [platform, code] of [ - ["darwin", "EPERM"], - ["win32", "EINVAL"], - ] as const) { - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-state-no-retry-", - }); - const calls = yield* Ref.make(0); - const injectedFs = Layer.effect( - FileSystem.FileSystem, - Effect.succeed({ - ...fs, - rename: (from: string, to: string) => - Ref.update(calls, (count) => count + 1).pipe( - Effect.andThen( - Effect.fail( - PlatformError.systemError({ - _tag: "Unknown", - module: "FileSystem", - method: "rename", - pathOrDescriptor: to, - cause: Object.assign(new Error("injected failure"), { - code, - }), - }), - ), - ), - ), - }), - ); - const store = yield* Layer.build( - State.layer({ root, platform }).pipe(Layer.provide(injectedFs)), - ).pipe(Effect.map((context) => Context.get(context, State.Service))); - const error = yield* store.save(initial).pipe(Effect.flip); - expect(error.operation).toBe("publish"); - expect(yield* Ref.get(calls)).toBe(1); - expect( - (yield* fs.readDirectory(root)).some((entry) => entry.startsWith(".state-write-")), - ).toBe(false); - } - }), - ), - ); - - const listingWithReadFailures = (options: { - readonly root: string; - readonly platform: NodeJS.Platform; - readonly code: string; - readonly failures: number; - }) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const target = (yield* Path.Path).join(options.root, initial.id, "state.json"); - const remaining = yield* Ref.make(options.failures); - const firstFailure = yield* Deferred.make(); - const reported = yield* Ref.make>([]); - const injectedFs = Layer.succeed(FileSystem.FileSystem, { - ...fs, - readFileString: (file: string, encoding?: string) => - Effect.gen(function* () { - if (file === target && (yield* Ref.get(remaining)) > 0) { - yield* Ref.update(remaining, (count) => count - 1); - yield* Deferred.succeed(firstFailure, undefined); - return yield* PlatformError.systemError({ - _tag: "Unknown", - module: "FileSystem", - method: "readFile", - pathOrDescriptor: file, - cause: Object.assign(new Error("injected read failure"), { code: options.code }), - }); - } - return yield* fs.readFileString(file, encoding); - }), - }); - const store = yield* Layer.build( - State.layer({ - root: options.root, - platform: options.platform, - onInvalidState: (id) => Ref.update(reported, (ids) => [...ids, id]), - }).pipe(Layer.provide(injectedFs)), - ).pipe(Effect.map((context) => Context.get(context, State.Service))); - return { store, firstFailure, reported }; - }); - - it.effect("lists a stack after a transient Windows read failure clears", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-list-retry-" }); - yield* (yield* makeTestState(root)).save(initial); - const { store, firstFailure, reported } = yield* listingWithReadFailures({ - root, - platform: "win32", - code: "EBUSY", - failures: 1, - }); - - const listing = yield* store.list.pipe(Effect.forkScoped); - yield* Deferred.await(firstFailure); - yield* TestClock.adjust("10 millis"); - expect((yield* Fiber.join(listing)).map(({ id }) => id)).toEqual([initial.id]); - expect(yield* Ref.get(reported)).toEqual([]); - }), - ), - ); - - it.effect("skips and reports a stack whose state stays unreadable after retries", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - for (const [platform, code] of [ - ["win32", "EBUSY"], - ["linux", "EACCES"], - ] as const) { - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-list-skip-" }); - yield* (yield* makeTestState(root)).save(initial); - const { store, firstFailure, reported } = yield* listingWithReadFailures({ - root, - platform, - code, - failures: Number.POSITIVE_INFINITY, - }); - - const listing = yield* store.list.pipe(Effect.forkScoped); - yield* Deferred.await(firstFailure); - yield* TestClock.adjust("950 millis"); - expect(yield* Fiber.join(listing)).toEqual([]); - expect(yield* Ref.get(reported)).toEqual([initial.id]); - } - }), - ), - ); - - it.live.skipIf(process.platform === "win32" || process.getuid?.() === 0)( - "lists and claims readable stacks past a sibling directory it cannot access", - () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-list-eacces-" }); - const reported: Array = []; - const store = yield* Layer.build( - State.layer({ root, onInvalidState: (id) => Effect.sync(() => reported.push(id)) }), - ).pipe(Effect.map((context) => Context.get(context, State.Service))); - yield* store.save(initial); - const locked = path.join(root, "locked"); - yield* fs.makeDirectory(locked, { mode: 0o700 }); - yield* fs.writeFileString(path.join(locked, "state.json"), "{}"); - yield* Effect.acquireRelease(fs.chmod(locked, 0o000), () => - fs.chmod(locked, 0o700).pipe(Effect.orDie), - ); - - expect((yield* store.list).map(({ id }) => id)).toEqual([initial.id]); - expect(reported).toEqual(["locked"]); - expect((yield* store.claims).map(({ id }) => id)).toEqual([initial.id]); - }), - ), - ); - - it.live("skips and reports malformed, mismatched, and non-file entries while listing", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-list-invalid-" }); - const reported: Array = []; - const store = yield* Layer.build( - State.layer({ - root, - onInvalidState: (id) => Effect.sync(() => reported.push(id)), - }), - ).pipe(Effect.map((context) => Context.get(context, State.Service))); - yield* store.save(initial); - yield* fs.makeDirectory(path.join(root, "malformed")); - yield* fs.writeFileString(path.join(root, "malformed", "state.json"), "{broken"); - yield* fs.makeDirectory(path.join(root, "mismatched")); - yield* fs.writeFileString( - path.join(root, "mismatched", "state.json"), - yield* Schema.encodeEffect(Schema.fromJsonString(State.SavedStack))(initial), - ); - yield* fs.makeDirectory(path.join(root, "directory", "state.json"), { recursive: true }); - yield* fs.writeFileString(path.join(root, "stray-file"), ""); - - expect((yield* store.list).map(({ id }) => id)).toEqual([initial.id]); - // Windows resolves a path below a regular file as missing rather than unreadable. - expect(reported.toSorted()).toEqual( - process.platform === "win32" - ? ["directory", "malformed", "mismatched"] - : ["directory", "malformed", "mismatched", "stray-file"], - ); - }), - ), - ); - - it.live("decodes saved compositions, creations, and instance ids strictly", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-typed-" }); - const reported: Array = []; - const store = yield* Layer.build( - State.layer({ - root, - onInvalidState: (id) => Effect.sync(() => reported.push(id)), - }), - ).pipe(Effect.map((context) => Context.get(context, State.Service))); - yield* store.save(initial); - const mail = { service: "mail", config: {} }; - const invalid = { - "bad-composition": { composition: { members: "none", dependencies: [] } }, - "bad-creation": { instances: [{ id: "one", creation: { service: "unknown" } }] }, - "duplicate-ids": { - instances: [ - { id: "one", creation: mail }, - { id: "one", creation: mail }, - ], - }, - }; - for (const [id, override] of Object.entries(invalid)) { - yield* fs.makeDirectory(path.join(root, id)); - yield* fs.writeFileString( - path.join(root, id, "state.json"), - yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))({ - ...initial, - id, - ...override, - }), - ); - } - - expect((yield* store.list).map(({ id }) => id)).toEqual([initial.id]); - expect(reported.toSorted()).toEqual(Object.keys(invalid)); - for (const id of Object.keys(invalid)) { - const failure = yield* store.read(id).pipe(Effect.flip); - expect(failure.operation).toBe("decode"); - expect(failure.message).toContain("Unable to decode state"); - expect(failure.message).toContain(`for stack ${id}`); - } - const duplicate = yield* store.read("duplicate-ids").pipe(Effect.flip); - expect(duplicate.message).toContain("Expected unique instance ids"); - }), - ), - ); - - it.effect("cleans up a cancelled Windows replacement and releases the state lock", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-state-cancel-rename-", - }); - const calls = yield* Ref.make(0); - const failRename = yield* Ref.make(false); - const firstFailure = yield* Deferred.make(); - const injectedFs = Layer.effect( - FileSystem.FileSystem, - Effect.succeed({ - ...fs, - rename: (from: string, to: string) => - Effect.gen(function* () { - if (from.endsWith("state.json") && to.endsWith("state.json")) { - yield* Ref.update(calls, (count) => count + 1); - if (yield* Ref.get(failRename)) { - yield* Deferred.succeed(firstFailure, undefined); - return yield* PlatformError.systemError({ - _tag: "Busy", - module: "FileSystem", - method: "rename", - pathOrDescriptor: to, - cause: Object.assign(new Error("sharing violation"), { - code: "EBUSY", - }), - }); - } - } - return yield* fs.rename(from, to); - }), - }), - ); - const store = yield* Layer.build( - State.layer({ root, platform: "win32" }).pipe(Layer.provide(injectedFs)), - ).pipe(Effect.map((context) => Context.get(context, State.Service))); - yield* store.save(initial); - yield* Ref.set(calls, 0); - yield* Ref.set(failRename, true); - const next = { - ...initial, - identity: { ...initial.identity, stackName: "next" }, - }; - const saving = yield* store.withLock(store.save(next)).pipe(Effect.forkScoped); - yield* Deferred.await(firstFailure); - yield* TestClock.adjust("1 millis"); - expect(yield* Ref.get(calls)).toBe(1); - yield* Fiber.interrupt(saving); - expect(yield* store.read(initial.id)).toEqual(initial); - expect( - (yield* fs.readDirectory(root)).some((entry) => entry.startsWith(".state-write-")), - ).toBe(false); - - yield* Ref.set(failRename, false); - yield* store.withLock(store.save(next)); - expect((yield* store.read(initial.id))?.identity.stackName).toBe("next"); - }), - ), - ); - - it.live("reaps write leftovers of dead writers and keeps recent ones on open", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-reap-" }); - const abandoned = path.join(root, ".state-write-abandoned"); - const inProgress = path.join(root, ".state-write-in-progress"); - for (const directory of [abandoned, inProgress]) { - yield* fs.makeDirectory(directory); - yield* fs.writeFileString(path.join(directory, "state.json"), "{}"); - } - const twoDaysAgo = DateTime.toDateUtc(DateTime.subtract(yield* DateTime.now, { days: 2 })); - yield* fs.utimes(abandoned, twoDaysAgo, twoDaysAgo); - - const store = yield* makeTestState(root); - - expect(yield* fs.exists(abandoned)).toBe(false); - expect(yield* fs.exists(path.join(inProgress, "state.json"))).toBe(true); - yield* store.save(initial); - expect(yield* store.read(initial.id)).toEqual(initial); - }), - ), - ); - - it.live("removes empty stack parents without deleting unowned data", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-remove-" }); - const store = yield* makeTestState(root); - yield* store.save(initial); - yield* fs.makeDirectory(path.join(root, initial.id, "data"), { recursive: true }); - yield* store.remove(initial.id); - expect(yield* fs.exists(path.join(root, initial.id))).toBe(false); - - yield* store.save(initial); - yield* fs.makeDirectory(path.join(root, initial.id, "data"), { recursive: true }); - yield* fs.writeFileString(path.join(root, initial.id, "data", "owned-by-caller"), "keep"); - yield* store.remove(initial.id); - expect(yield* fs.exists(path.join(root, initial.id, "data", "owned-by-caller"))).toBe(true); - }), - ), - ); - - it.live.skipIf(process.platform === "win32")( - "restricts the state root to its owner while keeping a traverse-only grant", - () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-traverse-" }); - yield* fs.chmod(root, 0o755); - yield* makeTestState(root); - expect((yield* fs.stat(root)).mode & 0o777).toBe(0o701); - }), - ), - ); - - it.effect("keeps a lock owner protected when a waiting fiber is cancelled", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-lock-" }); - const first = yield* makeTestState(root); - const second = yield* makeTestState(root); - const entered = yield* Deferred.make(); - const release = yield* Deferred.make(); - const owner = yield* Effect.forkScoped( - first.withLock( - Effect.gen(function* () { - yield* Deferred.succeed(entered, undefined); - yield* Deferred.await(release); - }), - ), - ); - yield* Deferred.await(entered); - const waiter = yield* Effect.forkScoped(second.withLock(second.save(initial))); - yield* TestClock.adjust("100 millis"); - yield* Fiber.interrupt(waiter); - expect(yield* second.read(initial.id)).toBeUndefined(); - const later = yield* Effect.forkScoped(second.withLock(second.save(initial))); - yield* TestClock.adjust("100 millis"); - expect(yield* second.read(initial.id)).toBeUndefined(); - yield* Deferred.succeed(release, undefined); - yield* Fiber.join(owner); - yield* TestClock.adjust("50 millis"); - yield* Fiber.join(later); - expect(yield* second.read(initial.id)).toEqual(initial); - }), - ), - ); - it.effect("times out contention without releasing the holder's lock", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-lock-timeout-" }); - const first = yield* makeTestState(root); - const second = yield* makeTestState(root); - const entered = yield* Deferred.make(); - const owner = yield* first - .withLock(Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never))) - .pipe(Effect.forkScoped); - yield* Deferred.await(entered); - const waiter = yield* second - .withLock(second.save(initial)) - .pipe(Effect.flip, Effect.forkScoped); - yield* TestClock.adjust("6 seconds"); - const result = yield* Fiber.join(waiter); - expect(result.operation).toBe("lock"); - expect(result.message).toBe("Stack registry is locked by another operation; retry shortly"); - expect(yield* second.read(initial.id)).toBeUndefined(); - yield* Fiber.interrupt(owner); - yield* second.withLock(second.save(initial)); - expect(yield* second.read(initial.id)).toEqual(initial); - }), - ), - ); - - it.live("releases ownership after failure and defects without replacing the lock file", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-lock-failure-" }); - const state = yield* makeTestState(root); - const failure = new State.StateError({ operation: "test", message: "body failed" }); - expect(yield* state.withLock(Effect.fail(failure)).pipe(Effect.flip)).toBe(failure); - const defect = yield* state.withLock(Effect.die("body defect")).pipe(Effect.exit); - expect(Exit.hasDies(defect)).toBe(true); - yield* state.withLock(state.save(initial)); - expect(yield* state.read(initial.id)).toEqual(initial); - expect((yield* fs.readDirectory(root)).sort()).toEqual([ - ".registry-lock.sqlite", - initial.id, - ]); - expect((yield* fs.stat(`${root}/.registry-lock.sqlite`)).size).toBe(0n); - }), - ), - ); - it.effect("fails invalid lock files without retrying or changing saved state", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-lock-invalid-" }); - const state = yield* makeTestState(root); - yield* fs.writeFileString(`${root}/.registry-lock.sqlite`, "not a database"); - const error = yield* state.withLock(state.save(initial)).pipe(Effect.flip); - expect(error).toBeInstanceOf(State.StateError); - expect(error.operation).toBe("lock"); - expect(yield* state.read(initial.id)).toBeUndefined(); - }), - ), - ); -}); - -describe("stack owner lease", () => { - it.live("hands the lease of a removed stack to a waiter on a live lease file", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-lease-handoff-" }); - const holder = yield* makeTestState(root); - const contended = yield* Deferred.make(); - const waiter = Context.get( - yield* Layer.build( - State.layer({ - root, - onLeaseContended: () => Deferred.succeed(contended, undefined).pipe(Effect.asVoid), - }), - ), - State.Service, - ); - const observer = yield* makeTestState(root); - const holderScope = yield* Scope.make(); - expect(yield* holder.lease("gone").pipe(Scope.provide(holderScope))).toBe(true); - - const waiterScope = yield* Scope.make(); - const waiting = yield* waiter - .lease("gone") - .pipe(Scope.provide(waiterScope), Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(contended); - yield* Scope.close(holderScope, Exit.void); - - expect(yield* Fiber.join(waiting)).toBe(true); - expect(yield* observer.leased("gone"), "the path names the file the waiter locked").toBe( - true, - ); - yield* Scope.close(waiterScope, Exit.void); - expect(yield* observer.leased("gone")).toBe(false); - expect(yield* fs.exists(`${root}/gone`), "the last holder removes the directory").toBe( - false, - ); - }), - ), - ); - - it.live("reports a free lease without creating a lease file", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-lease-probe-" }); - const state = yield* makeTestState(root); - yield* state.save(initial); - expect(yield* state.leased(initial.id)).toBe(false); - expect(yield* state.readHolder(initial.id), "a retracted record reads as absent").toBe( - undefined, - ); - expect(yield* fs.exists(`${root}/${initial.id}/owner.lock`)).toBe(false); - }), - ), - ); -}); diff --git a/packages/stack/src/State.process.integration.test.ts b/packages/stack/src/State.process.integration.test.ts deleted file mode 100644 index 90dec13572..0000000000 --- a/packages/stack/src/State.process.integration.test.ts +++ /dev/null @@ -1,208 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { expect, it } from "@effect/vitest"; -import { - Context, - Data, - Deferred, - Effect, - Fiber, - FileSystem, - Layer, - Ref, - Schema, - Stream, -} from "effect"; -import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { fileURLToPath } from "node:url"; -import { ownerExitProbe, waitForOwnerExit } from "./HostProcess.ts"; -import * as State from "./State.ts"; - -class FixtureError extends Data.TaggedError("StateLockFixtureError")<{ - readonly message: string; -}> {} -const fixture = fileURLToPath(new URL("../tests/state-lock-fixture.ts", import.meta.url)); -const readOutput = (child: ChildProcessSpawner.ChildProcessHandle) => - Effect.all( - [ - child.stdout.pipe(Stream.decodeText, Stream.mkString), - child.stderr.pipe(Stream.decodeText, Stream.mkString), - child.exitCode, - ], - { concurrency: "unbounded" }, - ); - -for (const compiled of [false, true]) { - it.live( - `serializes writers and releases a killed holder while its child survives (${compiled ? "compiled" : "source"})`, - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-lock-process-" }); - const executable = compiled - ? `${root}/fixture${process.platform === "win32" ? ".exe" : ""}` - : process.execPath; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const spawn = (command: string, args: ReadonlyArray) => - spawner.spawn( - ChildProcess.make(command, args, { - stdin: "ignore", - stdout: "pipe", - stderr: "pipe", - detached: true, - forceKillAfter: "1 second", - }), - ); - if (compiled) { - const build = yield* spawn("bun", [ - "build", - fixture, - "--compile", - `--outfile=${executable}`, - ]); - const [, stderr, code] = yield* readOutput(build); - expect(code, stderr).toBe(0); - } - const stateRoot = `${root}/state`; - const context = yield* Layer.build(State.layer({ root: stateRoot })); - const state = Context.get(context, State.Service); - const saved: State.SavedStack = { - id: "stack-main", - identity: { projectRoot: root, branchContext: "test", stackName: "lock" }, - runtime: "native", - instances: [], - lifetime: "detached", - composition: { members: [], dependencies: [] }, - ports: [], - }; - yield* state.save(saved); - const args = (mode: string, id?: string) => [ - ...(compiled ? [] : [fixture]), - mode, - stateRoot, - ...(id === undefined ? [] : [id]), - ]; - yield* Effect.forEach( - ["first", "second", "third"], - (id) => - Effect.scoped( - Effect.gen(function* () { - const writer = yield* spawn(executable, args("write", id)); - const [, stderr, code] = yield* readOutput(writer); - expect(code, stderr).toBe(0); - }), - ), - { concurrency: "unbounded" }, - ); - expect((yield* state.read(saved.id))?.instances.map(({ id }) => id).sort()).toEqual([ - "first", - "second", - "third", - ]); - - const holder = yield* spawn(executable, args("hold")); - const locked = yield* Deferred.make(); - const childReady = yield* Deferred.make(); - const observed = yield* Ref.make>([]); - const output = yield* holder.stdout.pipe( - Stream.decodeText, - Stream.splitLines, - Stream.runForEach((line) => - Ref.update(observed, (lines) => [...lines, line]).pipe( - Effect.andThen( - line === "locked" - ? Deferred.succeed(locked, undefined).pipe(Effect.asVoid) - : line.startsWith("child:") - ? Deferred.succeed(childReady, line.slice(6)).pipe(Effect.asVoid) - : Effect.void, - ), - ), - ), - Effect.forkScoped, - ); - const stderr = yield* Ref.make(""); - const diagnostics = yield* holder.stderr.pipe( - Stream.decodeText, - Stream.runForEach((chunk) => Ref.update(stderr, (text) => text + chunk)), - Effect.forkScoped, - ); - yield* Effect.all([Deferred.await(locked), Deferred.await(childReady)]).pipe( - Effect.raceFirst( - holder.exitCode.pipe( - Effect.matchEffect({ - onFailure: (cause) => - Ref.get(stderr).pipe( - Effect.flatMap((text) => - Effect.fail( - new FixtureError({ - message: `Holder exited before readiness: ${String(cause)}\n${text}`, - }), - ), - ), - ), - onSuccess: (code) => - Ref.get(stderr).pipe( - Effect.flatMap((text) => - Effect.fail( - new FixtureError({ - message: `Holder exited before readiness (${code}): ${text}`, - }), - ), - ), - ), - }), - ), - ), - Effect.timeoutOrElse({ - duration: "10 seconds", - orElse: () => - Effect.all([Ref.get(stderr), Ref.get(observed)]).pipe( - Effect.flatMap(([text, lines]) => - Effect.fail( - new FixtureError({ - message: `Holder readiness timed out: ${lines.join("\n")}\n${text}`, - }), - ), - ), - ), - }), - ); - const childPid = yield* Schema.decodeEffect(Schema.FiniteFromString)( - yield* Deferred.await(childReady), - ); - const kill = (pid: number) => - Effect.try({ - try: () => process.kill(pid, "SIGKILL"), - catch: (cause) => new FixtureError({ message: String(cause) }), - }); - yield* Effect.addFinalizer(() => - kill(childPid).pipe( - Effect.ignore, - Effect.andThen(Fiber.join(output)), - Effect.andThen(Fiber.join(diagnostics)), - // Windows keeps the compiled child's image in the temp root locked until it exits. - Effect.andThen(waitForOwnerExit(childPid, ownerExitProbe(fs)).pipe(Effect.ignore)), - Effect.timeout("10 seconds"), - Effect.orDie, - ), - ); - yield* kill(Number(holder.pid)); - yield* holder.exitCode.pipe(Effect.exit); - yield* state.withLock( - state.save({ ...saved, identity: { ...saved.identity, stackName: "after-crash" } }), - ); - expect((yield* state.read(saved.id))?.identity.stackName).toBe("after-crash"); - yield* Effect.try({ - try: () => process.kill(childPid, 0), - catch: (cause) => - new FixtureError({ message: `Child did not survive holder: ${String(cause)}` }), - }); - expect((yield* fs.stat(`${stateRoot}/.registry-lock.sqlite`)).size).toBe(0n); - expect((yield* fs.readDirectory(stateRoot)).sort()).toEqual([ - ".registry-lock.sqlite", - saved.id, - ]); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); -} diff --git a/packages/stack/src/State.ts b/packages/stack/src/State.ts deleted file mode 100644 index 7b738ac908..0000000000 --- a/packages/stack/src/State.ts +++ /dev/null @@ -1,597 +0,0 @@ -import { - Clock, - Data, - Duration, - Effect, - Exit, - FileSystem, - Context, - Layer, - Option, - Path, - Predicate, - Schedule, - Schema, - Scope, -} from "effect"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no non-recursive directory removal operation. -import { rmdir } from "node:fs/promises"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no OS-owned cross-process lock primitive. -import { DatabaseSync } from "node:sqlite"; -import { pathToFileURL } from "node:url"; -import { CompositionConfig } from "./Orchestrator.ts"; -import { restrictDirectoryToOwner } from "./runtime/postgres-user.ts"; -import { ServiceCreation } from "./services/Catalog.ts"; - -const SafeId = Schema.String.pipe( - Schema.refine((value): value is string => /^[a-zA-Z0-9_-]+$/u.test(value), { - identifier: "SafeStateId", - message: "Expected a safe state id", - }), -); - -const SavedInstance = Schema.Struct({ - id: SafeId, - creation: Schema.toCodecJson(ServiceCreation), -}); -interface SavedInstance extends Schema.Schema.Type {} - -/** API and JWT signing keys that override the keys a stack derives by default. */ -export const StackKeysInput = Schema.Struct({ - publishableKey: Schema.optionalKey(Schema.String), - secretKey: Schema.optionalKey(Schema.String), - anonKey: Schema.optionalKey(Schema.String), - anonKeyIsOverride: Schema.optionalKey(Schema.Boolean), - serviceRoleKey: Schema.optionalKey(Schema.String), - serviceRoleKeyIsOverride: Schema.optionalKey(Schema.Boolean), - gotrueJwtKeys: Schema.optionalKey(Schema.String), - publicSigningKeys: Schema.optionalKey(Schema.String), - remoteJwks: Schema.optionalKey(Schema.String), -}); -export interface StackKeysInput extends Schema.Schema.Type {} - -export const StackCredentials = Schema.Struct({ - jwtSecret: Schema.String, - postgresRootKey: Schema.String, - databasePassword: Schema.String, - publishableKey: Schema.String, - secretKey: Schema.String, - anonKey: Schema.String, - serviceRoleKey: Schema.String, - jwks: Schema.String, - gotrueJwtKeys: Schema.String, - remoteJwks: Schema.String, - anonKeyIsOverride: Schema.Boolean, - serviceRoleKeyIsOverride: Schema.Boolean, -}); -export interface StackCredentials extends Schema.Schema.Type {} - -const PortClaim = Schema.Struct({ - key: Schema.String, - host: Schema.String, - port: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 65535 })), -}); -export interface PortClaim extends Schema.Schema.Type {} - -/** A session stack is destroyed when its creator exits; a detached stack outlives it. */ -export const StackLifetime = Schema.Literals(["session", "detached"]); -export type StackLifetime = Schema.Schema.Type; - -export const SavedStack = Schema.Struct({ - id: SafeId, - lifetime: StackLifetime, - identity: Schema.Struct({ - projectRoot: Schema.String, - branchContext: Schema.String, - stackName: Schema.String, - }), - runtime: Schema.Literals(["native", "docker", "podman"]), - instances: Schema.Array(SavedInstance).check( - Schema.makeFilter((instances) => - new Set(instances.map(({ id }) => id)).size === instances.length - ? undefined - : "Expected unique instance ids", - ), - ), - composition: CompositionConfig, - credentials: Schema.optionalKey(StackCredentials), - ports: Schema.Array(PortClaim), -}); -export interface SavedStack extends Schema.Schema.Type {} - -const ClaimsDocument = Schema.Struct({ ports: Schema.Array(PortClaim) }); - -/** What the current lease holder publishes: an owner's control endpoint, or a sweeper's marker. */ -const LeaseHolder = Schema.Union([ - Schema.Struct({ - role: Schema.Literal("owner"), - port: PortClaim.fields.port, - pid: Schema.Int, - release: Schema.String, - lifetime: StackLifetime, - startedAt: Schema.String, - secret: Schema.String, - }), - Schema.Struct({ role: Schema.Literal("sweeper"), pid: Schema.Int, startedAt: Schema.String }), -]); -type LeaseHolder = Schema.Schema.Type; - -export interface StackClaims { - readonly id: string; - readonly ports: ReadonlyArray; -} - -export class StateError extends Data.TaggedError("StateError")<{ - readonly operation: string; - readonly message: string; - readonly cause?: unknown; -}> {} - -export interface Interface { - readonly read: (id: string) => Effect.Effect; - /** Skips each stack entry that stays unreadable after transient retries, reporting it to `onInvalidState`. */ - readonly list: Effect.Effect, StateError>; - /** Decodes only each stack's port claims and silently skips entries that cannot provide them. */ - readonly claims: Effect.Effect, StateError>; - readonly save: (state: SavedStack) => Effect.Effect; - readonly remove: (id: string) => Effect.Effect; - /** Not reentrant; wrap metadata updates here, while Ports operations acquire this lock themselves. */ - readonly withLock: ( - effect: Effect.Effect, - ) => Effect.Effect; - /** - * Holds the stack's owner lease for the enclosing scope, or returns `false` while another - * process holds it. Releasing the lease of a removed stack deletes its directory. - */ - readonly lease: (id: string) => Effect.Effect; - /** Reports whether any process currently holds the stack's owner lease. */ - readonly leased: (id: string) => Effect.Effect; - /** Only meaningful while the lease is held; a record left by a dead holder is stale. */ - readonly readHolder: (id: string) => Effect.Effect; - /** Written by the lease holder with owner-only permissions, since it carries the owner secret. */ - readonly publishHolder: (id: string, record: LeaseHolder) => Effect.Effect; - readonly retractHolder: (id: string) => Effect.Effect; - /** The file that receives the stack owner's stdout and stderr. */ - readonly ownerLog: (id: string) => string; -} - -export class Service extends Context.Service()("@supabase/stack/State") {} - -const stateError = (operation: string, cause: unknown): StateError => - new StateError({ - operation, - message: cause instanceof Error ? cause.message : String(cause), - cause, - }); - -const checkId = (id: string): Effect.Effect => - Schema.is(SafeId)(id) - ? Effect.void - : Effect.fail(stateError("identity", `Invalid state id: ${id}`)); - -const decodeState = ( - text: string, - id: string, - target: string, -): Effect.Effect => - Schema.decodeEffect(Schema.fromJsonString(SavedStack))(text).pipe( - Effect.mapError( - (cause) => - new StateError({ - operation: "decode", - message: `Unable to decode state ${target} for stack ${id}: ${cause.message}`, - cause, - }), - ), - ); - -interface Options { - readonly root: string; - readonly platform?: NodeJS.Platform; - readonly onInvalidState?: (id: string, error: StateError) => Effect.Effect; - /** Observes a lease request that found the lease held and is waiting for it. */ - readonly onLeaseContended?: (id: string) => Effect.Effect; -} - -const stateWritePrefix = ".state-write-"; -/** A state write takes milliseconds, so a temporary directory this old belongs to a dead writer. */ -const staleWriteAgeMillis = 24 * 60 * 60 * 1000; - -/** Best-effort removal of temporary write directories a killed writer left behind. */ -const reapStaleWrites = (fs: FileSystem.FileSystem, path: Path.Path, root: string) => - Effect.gen(function* () { - const now = yield* Clock.currentTimeMillis; - const entries = yield* fs.readDirectory(root); - yield* Effect.forEach( - entries.filter((entry) => entry.startsWith(stateWritePrefix)), - (entry) => { - const candidate = path.join(root, entry); - return fs.stat(candidate).pipe( - Effect.flatMap((info) => - Option.exists(info.mtime, (mtime) => now - mtime.getTime() > staleWriteAgeMillis) - ? fs.remove(candidate, { recursive: true, force: true }) - : Effect.void, - ), - Effect.ignore, - ); - }, - { discard: true }, - ); - }).pipe(Effect.ignore); - -const makeState = ( - options: Options, -): Effect.Effect => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = path.normalize(options.root); - // SQLite alone opens this persistent file; other descriptors can invalidate its POSIX locks. - const lock = path.join(root, ".registry-lock.sqlite"); - yield* fs - .makeDirectory(root, { recursive: true }) - .pipe(Effect.mapError((cause) => stateError("root", cause))); - yield* restrictDirectoryToOwner(fs, root).pipe( - Effect.mapError((cause) => stateError("root", cause)), - ); - yield* reapStaleWrites(fs, path, root); - - const stackRoot = (id: string) => path.join(root, id); - const statePath = (id: string) => path.join(stackRoot(id), "state.json"); - // Only lease code opens a lease file, for the same reason as the registry lock. - const leasePath = (id: string) => path.join(stackRoot(id), "owner.lock"); - const ownerPath = (id: string) => path.join(stackRoot(id), "owner.json"); - const ownerLog = (id: string) => path.join(stackRoot(id), "owner.log"); - const publishRetrySchedule = Schedule.exponential("10 millis", 2).pipe( - Schedule.modifyDelay(({ duration }) => - Effect.succeed(Duration.min(duration, Duration.millis(100))), - ), - Schedule.upTo({ times: 12 }), - ); - const errorCode = (error: unknown): string | undefined => { - if (!Predicate.hasProperty(error, "cause")) return undefined; - return Predicate.hasProperty(error.cause, "code") && typeof error.cause.code === "string" - ? error.cause.code - : undefined; - }; - /** Windows reports a file that another process is replacing as a transient sharing violation. */ - const sharingViolation = (error: unknown) => - (options.platform ?? process.platform) === "win32" && - ["EPERM", "EACCES", "EBUSY"].includes(errorCode(error) ?? ""); - const retryTransientRead = (effect: Effect.Effect) => - effect.pipe( - Effect.retry({ schedule: publishRetrySchedule, while: sharingViolation }), - Effect.mapError((cause) => stateError("read", cause)), - ); - const publish = Effect.fn("State.publish")(function* (temporary: string, target: string) { - yield* fs.rename(temporary, target).pipe( - Effect.retry({ schedule: publishRetrySchedule, while: sharingViolation }), - Effect.mapError( - (cause) => - new StateError({ - operation: "publish", - message: `Unable to publish state to ${target}${errorCode(cause) ? ` (${errorCode(cause)})` : ""}: ${cause instanceof Error ? cause.message : String(cause)}`, - cause, - }), - ), - ); - }); - const removeEmptyDirectory = (directory: string) => - Effect.tryPromise({ - try: () => rmdir(directory), - catch: (cause) => stateError("cleanup", cause), - }).pipe( - Effect.catch((cause) => { - const code = - typeof cause.cause === "object" && cause.cause !== null && "code" in cause.cause - ? cause.cause.code - : undefined; - return code === "ENOENT" || code === "ENOTEMPTY" || code === "EEXIST" - ? Effect.void - : Effect.fail(cause); - }), - ); - const read = Effect.fn("State.read")(function* (id: string) { - yield* checkId(id); - const target = statePath(id); - const exists = yield* fs.exists(target).pipe(retryTransientRead); - if (!exists) return undefined; - const text = yield* fs.readFileString(target).pipe(retryTransientRead); - const state = yield* decodeState(text, id, target); - if (state.id !== id) { - return yield* stateError("identity", "State document identity does not match its path"); - } - return state; - }); - const stackIds = fs.readDirectory(root).pipe( - Effect.map((entries) => entries.filter(Schema.is(SafeId))), - Effect.mapError((cause) => stateError("list", cause)), - ); - const readEntries = ( - readEntry: (id: string) => Effect.Effect, - onSkipped: (id: string, error: StateError) => Effect.Effect, - ) => - Effect.gen(function* () { - const entries: Array = []; - for (const id of yield* stackIds) { - const value = yield* readEntry(id).pipe( - Effect.catch((error) => onSkipped(id, error).pipe(Effect.as(undefined))), - ); - if (value !== undefined) entries.push(value); - } - return entries; - }); - const list = Effect.fn("State.list")(() => - readEntries(read, (id, error) => options.onInvalidState?.(id, error) ?? Effect.void), - ); - const decodeClaims = Schema.decodeEffect(Schema.fromJsonString(ClaimsDocument)); - const readClaims = (id: string) => - fs.readFileString(statePath(id)).pipe( - retryTransientRead, - Effect.flatMap((text) => - decodeClaims(text).pipe(Effect.mapError((cause) => stateError("decode", cause))), - ), - Effect.map(({ ports }): StackClaims => ({ id, ports })), - ); - const claims = Effect.fn("State.claims")(() => readEntries(readClaims, () => Effect.void)); - const writeAtomically = (id: string, target: string, serialized: string) => - Effect.gen(function* () { - yield* fs - .makeDirectory(stackRoot(id), { recursive: true, mode: 0o700 }) - .pipe(Effect.mapError((cause) => stateError("write", cause))); - yield* Effect.acquireUseRelease( - fs - .makeTempDirectory({ directory: root, prefix: stateWritePrefix }) - .pipe(Effect.mapError((cause) => stateError("write", cause))), - (directory) => - Effect.gen(function* () { - const temporary = path.join(directory, path.basename(target)); - yield* fs - .writeFileString(temporary, serialized, { mode: 0o600 }) - .pipe(Effect.mapError((cause) => stateError("write", cause))); - yield* publish(temporary, target); - }), - (directory) => - fs - .remove(directory, { recursive: true, force: true }) - .pipe(Effect.mapError((cause) => stateError("cleanup", cause))), - ); - }); - const save = Effect.fn("State.save")(function* (state: SavedStack) { - yield* checkId(state.id); - const serialized = yield* Schema.encodeEffect(Schema.fromJsonString(SavedStack))(state).pipe( - Effect.mapError((cause) => stateError("encode", cause)), - ); - yield* writeAtomically(state.id, statePath(state.id), serialized); - }); - const remove = Effect.fn("State.remove")(function* (id: string) { - yield* checkId(id); - for (const file of [statePath(id), ownerPath(id), ownerLog(id)]) - yield* fs - .remove(file, { force: true }) - .pipe(Effect.mapError((cause) => stateError("remove", cause))); - yield* removeEmptyDirectory(path.join(stackRoot(id), "data")); - yield* removeEmptyDirectory(stackRoot(id)); - }); - const openLock = (file: string) => - Effect.try({ - try: () => new DatabaseSync(file), - catch: (cause) => stateError("lock", cause), - }); - /** Opens an existing lock file without creating a stray one. */ - const openExistingLock = (file: string) => - Effect.try({ - try: () => new DatabaseSync(new URL(`${pathToFileURL(file).href}?mode=rw`)), - catch: (cause) => stateError("lock", cause), - }); - const closeLock = (connection: DatabaseSync) => - Effect.try({ - try: () => connection.close(), - catch: (cause) => stateError("unlock", cause), - }); - const hasErrcode = (code: number) => (error: StateError) => - Predicate.hasProperty(error.cause, "errcode") && error.cause.errcode === code; - const isBusy = hasErrcode(5); - const isMissing = hasErrcode(14); - /** An open file that was unlinked: SQLite IOERR_VNODE on macOS, IOERR_FSTAT on Linux. */ - const isMoved = (error: StateError) => hasErrcode(6922)(error) || hasErrcode(1802)(error); - /** Takes the file's SQLite write lock on this connection, retrying contention on `schedule`. */ - const takeLock = ( - connection: DatabaseSync, - schedule: Schedule.Schedule, - onContended: Effect.Effect = Effect.void, - ) => - Effect.gen(function* () { - yield* Effect.try({ - try: () => connection.exec("PRAGMA busy_timeout = 0"), - catch: (cause) => stateError("lock", cause), - }); - let contended = false; - return yield* Effect.try({ - try: () => connection.exec("BEGIN IMMEDIATE"), - catch: (cause) => stateError("lock", cause), - }).pipe( - Effect.tapError((error) => - isBusy(error) && !contended - ? Effect.sync(() => { - contended = true; - }).pipe(Effect.andThen(onContended)) - : Effect.void, - ), - Effect.retry({ schedule, while: isBusy }), - Effect.as(true), - Effect.catchIf(isBusy, () => Effect.succeed(false)), - ); - }); - const withLock = Effect.fn("State.withLock")((effect: Effect.Effect) => - Effect.acquireUseRelease( - openLock(lock), - (connection) => - Effect.gen(function* () { - const held = yield* takeLock( - connection, - Schedule.spaced("50 millis").pipe(Schedule.upTo({ duration: "5 seconds" })), - ); - if (!held) - return yield* new StateError({ - operation: "lock", - message: "Stack registry is locked by another operation; retry shortly", - }); - return yield* effect; - }), - closeLock, - ), - ); - /** Deletes the lease file of an unregistered stack; `false` means it is still in place. */ - const removeLeaseFile = (id: string) => - fs.exists(statePath(id)).pipe( - Effect.flatMap((registered) => - registered ? Effect.void : fs.remove(leasePath(id), { force: true }), - ), - Effect.as(true), - Effect.orElseSucceed(() => false), - ); - const fileIdentity = (file: string) => - fs.stat(file).pipe( - Effect.map((info) => `${info.dev}:${Option.getOrElse(info.ino, () => "")}`), - Effect.option, - ); - /** - * A waiter can open the lease file just before its holder unlinks it, and then lock the - * unlinked file; comparing the path's file before opening and after locking rejects that. - * Unlinking under the lock keeps new openers off a doomed file. Windows refuses to unlink an - * open file, so only there does a second attempt follow the close. - */ - const lease = Effect.fn("State.lease")(function* (id: string) { - yield* checkId(id); - const target = leasePath(id); - const scope = yield* Scope.Scope; - let held = false; - let unlinked = false; - yield* Effect.addFinalizer(() => - held - ? (unlinked ? Effect.void : removeLeaseFile(id)).pipe( - Effect.andThen(removeEmptyDirectory(stackRoot(id))), - Effect.ignore, - ) - : Effect.void, - ); - const onContended = options.onLeaseContended?.(id) ?? Effect.void; - for (let attempt = 0; attempt < 8; attempt++) { - // A releasing holder may remove the empty stack directory at any moment. - yield* fs - .makeDirectory(stackRoot(id), { recursive: true, mode: 0o700 }) - .pipe(Effect.mapError((cause) => stateError("lease", cause))); - const before = yield* fileIdentity(target); - const attemptScope = yield* Scope.fork(scope, "sequential"); - const connection = yield* Effect.acquireRelease(openLock(target), (connection) => - closeLock(connection).pipe( - Effect.catch((error) => - Effect.logWarning(`Unable to release stack lease ${id}`, error), - ), - ), - ).pipe( - Scope.provide(attemptScope), - Effect.map(Option.some), - Effect.catchIf(isMissing, () => Effect.succeed(Option.none())), - ); - if (Option.isNone(connection)) { - yield* Scope.close(attemptScope, Exit.void); - continue; - } - const acquired = yield* takeLock( - connection.value, - Schedule.spaced("25 millis").pipe(Schedule.upTo({ duration: "500 millis" })), - onContended, - ).pipe( - Effect.map((held) => (held ? "held" : "busy")), - Effect.catchIf(isMoved, () => Effect.succeed("moved" as const)), - ); - if (acquired === "busy") { - yield* Scope.close(attemptScope, Exit.void); - return false; - } - const after = acquired === "moved" ? Option.none() : yield* fileIdentity(target); - if (Option.isSome(before) && Option.isSome(after) && before.value === after.value) { - held = true; - yield* Scope.addFinalizer( - attemptScope, - removeLeaseFile(id).pipe( - Effect.map((removed) => { - unlinked = removed; - }), - ), - ); - return true; - } - yield* Scope.close(attemptScope, Exit.void); - } - return yield* stateError("lease", `The lease file of stack ${id} keeps changing`); - }); - const leased = Effect.fn("State.leased")(function* (id: string) { - yield* checkId(id); - return yield* Effect.acquireUseRelease( - openExistingLock(leasePath(id)), - (connection) => - takeLock(connection, Schedule.recurs(0)).pipe(Effect.map((acquired) => !acquired)), - closeLock, - ).pipe( - // A lease file that is missing, or that its holder unlinked meanwhile, is not held. - Effect.catchIf( - (error) => isMissing(error) || isMoved(error), - () => Effect.succeed(false), - ), - ); - }); - const decodeHolder = Schema.decodeEffect(Schema.fromJsonString(LeaseHolder)); - const readHolder = Effect.fn("State.readHolder")(function* (id: string) { - yield* checkId(id); - const target = ownerPath(id); - // The holder may retract its record at any moment, so a missing record is not an error. - const text = yield* fs.readFileString(target).pipe( - Effect.map(Option.some), - Effect.catchIf( - (error) => error.reason._tag === "NotFound", - () => Effect.succeed(Option.none()), - ), - retryTransientRead, - ); - if (Option.isNone(text)) return undefined; - return yield* decodeHolder(text.value).pipe( - Effect.mapError((cause) => stateError("decode", `Unable to decode ${target}: ${cause}`)), - ); - }); - const publishHolder = Effect.fn("State.publishHolder")(function* ( - id: string, - record: LeaseHolder, - ) { - yield* checkId(id); - const serialized = yield* Schema.encodeEffect(Schema.fromJsonString(LeaseHolder))( - record, - ).pipe(Effect.mapError((cause) => stateError("encode", cause))); - yield* writeAtomically(id, ownerPath(id), serialized); - }); - const retractHolder = Effect.fn("State.retractHolder")(function* (id: string) { - yield* checkId(id); - yield* fs - .remove(ownerPath(id), { force: true }) - .pipe(Effect.mapError((cause) => stateError("remove", cause))); - }); - return { - read, - list: list(), - claims: claims(), - save, - remove, - withLock, - lease, - leased, - readHolder, - publishHolder, - retractHolder, - ownerLog, - }; - }); - -export const layer = (options: Options) => - Layer.effect(Service, makeState(options).pipe(Effect.map(Service.of))); diff --git a/packages/stack/src/Sweep.integration.test.ts b/packages/stack/src/Sweep.integration.test.ts index 70c1754af0..28a4c7536d 100644 --- a/packages/stack/src/Sweep.integration.test.ts +++ b/packages/stack/src/Sweep.integration.test.ts @@ -11,12 +11,15 @@ import { Layer, Option, Path, + Queue, Stream, } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- test-only native watcher for synchronous attachment, no Effect wrapper gives this guarantee. +import { watch as nodeWatch } from "node:fs"; import { launchHost } from "./HostProcess.ts"; import { makeContainerRuntime } from "./runtime/Container.ts"; -import * as State from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; import { shutdownOwner, watchLeaseRelease } from "../tests/owner.ts"; @@ -104,25 +107,42 @@ const awaitDestroyed = (id: string, since: number) => }), ); -const awaitRemoval = (directory: string, entry: string) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const target = path.join(directory, entry); - yield* fs.watch(directory).pipe( - Stream.filter((event) => event.path === entry || event.path === target), - Stream.mapEffect(() => fs.exists(target)), - Stream.takeUntil((exists) => !exists), - Stream.runDrain, - ); - }); +/** + * Watches `directory` and waits for `entry`'s removal, scoped to the caller's own fork so the + * watcher closes as soon as it returns. `node:fs.watch` itself is synchronous, so acquiring it + * directly (rather than through `Stream`, which forks registration through `Stream.callback`, + * `Channel.callbackArray`, and `asyncQueue` before it actually attaches) guarantees the watcher is + * live before this returns: no canary handshake needed. Node reports non-recursive watch events by + * bare filename and sometimes coalesces them, so any event re-checks the real directory instead of + * trusting which path it names. The watcher's native resource is torn down whether or not removal + * ever arrives, so this never succeeds by a wait simply running out. + */ +const awaitRemoval = Effect.fn("SweepTest.awaitRemoval")(function* ( + directory: string, + entry: string, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const target = path.join(directory, entry); + const queue = yield* Queue.unbounded(); + yield* Effect.acquireRelease( + Effect.sync(() => + nodeWatch(directory, (_event, filename) => { + if (filename) Queue.offerUnsafe(queue, filename); + }), + ), + (watcher) => Effect.sync(() => watcher.close()), + ); + + while (yield* fs.exists(target)) yield* Queue.take(queue); +}); const saved = ( id: string, projectRoot: string, - runtime: State.SavedStack["runtime"], - lifetime: State.StackLifetime, -): State.SavedStack => ({ + runtime: StackNamespace.SavedStack["runtime"], + lifetime: StackNamespace.StackLifetime, +): StackNamespace.SavedStack => ({ id, runtime, lifetime, @@ -152,7 +172,10 @@ it.live.skipIf(process.platform === "win32")( const cacheRoot = `${path.dirname(rootA)}/cache`; const helper = yield* makeContainerRuntime({ engine: "docker", root: dataA }); yield* helper.prepare(helperImage); - const state = Context.get(yield* Layer.build(State.layer({ root: rootA })), State.Service); + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root: rootA })), + StackNamespace.Service, + ); yield* state.save(saved(deadId, `${rootA}/dead`, "docker", "detached")); const dead = (yield* launchHost(state, { stateRoot: rootA, cacheRoot, stackId: deadId })) diff --git a/packages/stack/src/Sweep.ts b/packages/stack/src/Sweep.ts index 6817993f21..05e017ab51 100644 --- a/packages/stack/src/Sweep.ts +++ b/packages/stack/src/Sweep.ts @@ -1,12 +1,12 @@ import { Context, DateTime, Effect, FileSystem, Layer, Path } from "effect"; import { sweepTimeout } from "./HostProcess.ts"; import * as Owner from "./Owner.ts"; -import * as State from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; /** Runs a dead session stack's own destroy path in this process while holding its lease. */ const destroyStack = Effect.fn("Sweep.destroyStack")(function* ( - state: State.Interface, - saved: State.SavedStack, + state: StackNamespace.Interface, + saved: StackNamespace.SavedStack, dataRoot: string, cacheRoot: string, ) { @@ -14,7 +14,7 @@ const destroyStack = Effect.fn("Sweep.destroyStack")(function* ( yield* fs.makeDirectory(dataRoot, { recursive: true }); const context = yield* Layer.build( Owner.layer({ saved, root: dataRoot, cacheRoot }).pipe( - Layer.provide(Layer.succeed(State.Service, state)), + Layer.provide(Layer.succeed(StackNamespace.Service, state)), ), ); yield* Context.get(context, Owner.Service).namespace.destroy; @@ -27,7 +27,7 @@ const destroyStack = Effect.fn("Sweep.destroyStack")(function* ( * another process holds the lease. */ export const reclaimStack = Effect.fn("Sweep.reclaimStack")(function* (options: { - readonly state: State.Interface; + readonly state: StackNamespace.Interface; readonly stateRoot: string; readonly cacheRoot: string; readonly id: string; @@ -37,9 +37,12 @@ export const reclaimStack = Effect.fn("Sweep.reclaimStack")(function* (options: const { state, id } = options; return yield* Effect.scoped( Effect.gen(function* () { - if (!(yield* state.lease(id))) return false; - yield* Effect.addFinalizer(() => state.retractHolder(id).pipe(Effect.ignore)); - yield* state.publishHolder(id, { + const lease = yield* state + .acquireLease(id) + .pipe(Effect.catchTag("Namespace.LeaseHeldError", () => Effect.void)); + if (lease === undefined) return false; + yield* Effect.addFinalizer(() => lease.retractHolder.pipe(Effect.ignore)); + yield* lease.publishHolder({ role: "sweeper", pid: process.pid, startedAt: DateTime.formatIso(yield* DateTime.now), @@ -61,7 +64,7 @@ export const reclaimStack = Effect.fn("Sweep.reclaimStack")(function* (options: */ export const sweepOrphans = Effect.fn("Sweep.orphans")( function* (options: { - readonly state: State.Interface; + readonly state: StackNamespace.Interface; readonly stateRoot: string; readonly cacheRoot: string; readonly ownerId: string; diff --git a/packages/stack/src/composition/Supabase.native.integration.test.ts b/packages/stack/src/composition/Supabase.native.integration.test.ts index fffc8cc9b5..39bd792e11 100644 --- a/packages/stack/src/composition/Supabase.native.integration.test.ts +++ b/packages/stack/src/composition/Supabase.native.integration.test.ts @@ -16,8 +16,8 @@ import { PgClient } from "@effect/sql-pg"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { SignJWT } from "jose"; import { tmpdir } from "node:os"; -import * as State from "../State.ts"; -import type { SavedStack } from "../State.ts"; +import * as StackNamespace from "../StackNamespace.ts"; +import type { SavedStack } from "../StackNamespace.ts"; import { ownerFor } from "../../tests/owner-rpc.ts"; import { makeSupabaseComposition, @@ -32,8 +32,8 @@ const FIXED_MAIL_PORT = 24_392; const stateFor = (root: string) => Effect.gen(function* () { - const context = yield* Layer.build(State.layer({ root })); - return Context.get(context, State.Service); + const context = yield* Layer.build(StackNamespace.layer({ root })); + return Context.get(context, StackNamespace.Service); }); const query = (url: string, statement: string) => diff --git a/packages/stack/src/composition/Supabase.ts b/packages/stack/src/composition/Supabase.ts index 19fb3eeaf9..7c2c4431fa 100644 --- a/packages/stack/src/composition/Supabase.ts +++ b/packages/stack/src/composition/Supabase.ts @@ -3,7 +3,7 @@ import { postgresVersion } from "../Artifacts.ts"; import { causeMessage, type CompositionConfig } from "../Orchestrator.ts"; import type { Observation } from "../Rpc.ts"; import { ServiceCreation, type ServiceCreationInput } from "../services/Catalog.ts"; -import type { SavedStack, StackKeysInput } from "../State.ts"; +import type { SavedStack, StackKeysInput } from "../StackNamespace.ts"; import { credentialInputNames } from "../host/Credentials.ts"; import { apiRoute, endpointNames, endpointPort } from "../host/Endpoints.ts"; diff --git a/packages/stack/src/effect.integration.test.ts b/packages/stack/src/effect.integration.test.ts index 723bb24fc1..3a50b3f98c 100644 --- a/packages/stack/src/effect.integration.test.ts +++ b/packages/stack/src/effect.integration.test.ts @@ -28,7 +28,7 @@ import { initialization, postgres } from "./Commands.ts"; import { fileURLToPath } from "node:url"; import { launchHost } from "./HostProcess.ts"; import * as PromiseApi from "./index.ts"; -import * as State from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; import { assertOwnerExited, watchLeaseRelease } from "../tests/owner.ts"; import { foreignRelease } from "../tests/release-owner-fixture.ts"; import { destroyTestStack } from "../tests/stack-cleanup.ts"; @@ -439,8 +439,8 @@ it.live("rejects an owner of another release while stop and destroy still reach runtime: "native", } satisfies Parameters[0]; const stack = yield* create(options); - const state = yield* State.Service.pipe( - Effect.provide(State.layer({ root: options.stateRoot })), + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: options.stateRoot })), ); const startForeignOwner = launchHost(state, { ...options, @@ -475,14 +475,14 @@ it.live("treats a sweeper's hold as no owner and starts one once the sweep ends" runtime: "native", } satisfies Parameters[0]; const stack = yield* create(options); - const state = yield* State.Service.pipe( - Effect.provide(State.layer({ root: options.stateRoot })), + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: options.stateRoot })), ); yield* Effect.ensuring( Effect.gen(function* () { const sweep = yield* Scope.make(); - expect(yield* state.lease(stack.id).pipe(Scope.provide(sweep))).toBe(true); - yield* state.publishHolder(stack.id, { + const lease = yield* state.acquireLease(stack.id).pipe(Scope.provide(sweep)); + yield* lease.publishHolder({ role: "sweeper", pid: process.pid, startedAt: "2026-01-01T00:00:00.000Z", @@ -494,7 +494,7 @@ it.live("treats a sweeper's hold as no owner and starts one once the sweep ends" const starting = yield* stack.composition.start.pipe( Effect.forkChild({ startImmediately: true }), ); - yield* state.retractHolder(stack.id); + yield* lease.retractHolder; yield* Scope.close(sweep, Exit.void); expect(yield* Fiber.join(starting)).toEqual([]); expect((yield* discover(options))[0]?.host).toBeDefined(); @@ -515,14 +515,14 @@ it.live("interrupts a call waiting for an owner while a sweeper holds the stack" runtime: "native", } satisfies Parameters[0]; const stack = yield* create(options); - const state = yield* State.Service.pipe( - Effect.provide(State.layer({ root: options.stateRoot })), + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: options.stateRoot })), ); yield* Effect.ensuring( Effect.gen(function* () { const sweep = yield* Scope.make(); - expect(yield* state.lease(stack.id).pipe(Scope.provide(sweep))).toBe(true); - yield* state.publishHolder(stack.id, { + const lease = yield* state.acquireLease(stack.id).pipe(Scope.provide(sweep)); + yield* lease.publishHolder({ role: "sweeper", pid: process.pid, startedAt: "2026-01-01T00:00:00.000Z", @@ -531,7 +531,7 @@ it.live("interrupts a call waiting for an owner while a sweeper holds the stack" const waited = yield* stack.composition.start.pipe(Effect.timeoutOption("200 millis")); expect(Option.isNone(waited), "the wait for the sweeper ends at the timeout").toBe(true); - yield* state.retractHolder(stack.id); + yield* lease.retractHolder; yield* Scope.close(sweep, Exit.void); expect(yield* stack.composition.start, "the handle still launches afterwards").toEqual([]); }), @@ -600,8 +600,8 @@ it.live("sends no call to a process that took a dead owner's port", () => const stack = yield* create(options); const mail = yield* stack.services.create({ service: "mail", config: {} }); yield* mail.status; - const state = yield* State.Service.pipe( - Effect.provide(State.layer({ root: options.stateRoot })), + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: options.stateRoot })), ); const holder = yield* state.readHolder(stack.id); if (holder?.role !== "owner") return yield* Effect.die("Expected a live owner record"); @@ -631,8 +631,8 @@ it.live("sends no call to a process that took the port of a replaced owner", () const stack = yield* create(options); const mail = yield* stack.services.create({ service: "mail", config: {} }); yield* mail.status; - const state = yield* State.Service.pipe( - Effect.provide(State.layer({ root: options.stateRoot })), + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: options.stateRoot })), ); const replaced = yield* state.readHolder(stack.id); if (replaced?.role !== "owner") return yield* Effect.die("Expected a live owner record"); @@ -716,8 +716,8 @@ it.live("confirms owner exit after shutdown even while a stray handle keeps its runtime: "native", } satisfies Parameters[0]; const stack = yield* create(options); - const state = yield* State.Service.pipe( - Effect.provide(State.layer({ root: options.stateRoot })), + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: options.stateRoot })), ); const owner = yield* launchHost(state, { ...options, @@ -745,8 +745,8 @@ it.live("replaces a dead session stack that holds the requested identity", () => } satisfies Parameters[0]; const identity = yield* resolveStackIdentity(options); const id = yield* deriveStackId(identity); - const state = yield* State.Service.pipe( - Effect.provide(State.layer({ root: options.stateRoot })), + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: options.stateRoot })), ); yield* state.save({ id, diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index 40e09273f6..b30e4d4706 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -42,8 +42,8 @@ import { removeStackContainersCommand } from "./runtime/Container.ts"; import { volumeDataCleanupCommands } from "./storage/DockerDatabaseStorage.ts"; import { deriveStackId, resolveStackIdentity } from "./identity/Identity.ts"; import { failureMessage } from "./internal/failure-message.ts"; -import * as State from "./State.ts"; -import type { SavedStack, StackCredentials, StackKeysInput } from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; +import type { SavedStack, StackCredentials, StackKeysInput } from "./StackNamespace.ts"; import { StackError, type Definition, type Observation } from "./Rpc.ts"; import { reclaimStack } from "./Sweep.ts"; import { @@ -74,7 +74,7 @@ export type { SupabaseCompositionOptions, } from "./composition/Supabase.ts"; export { StackIdSchema as StackId } from "./identity/StackId.ts"; -export type { SavedStack } from "./State.ts"; +export type { SavedStack } from "./StackNamespace.ts"; export type { StackCredentials, StackKeysInput }; export type { Observation } from "./Rpc.ts"; export type { @@ -84,7 +84,8 @@ export type { PostgresCommand, } from "./Commands.ts"; -const stateFor = (root: string) => State.Service.pipe(Effect.provide(State.layer({ root }))); +const stateFor = (root: string) => + StackNamespace.Service.pipe(Effect.provide(StackNamespace.layer({ root }))); /** Storage locations shared by clients and the detached stack owner. */ export interface StackLocations { @@ -100,7 +101,7 @@ export interface CreateOptions extends StackLocations { * A `session` stack starts its owner at creation and is destroyed when the creating handle's * scope closes or its process exits; a `detached` stack (the default) outlives its creator. */ - readonly lifetime?: State.StackLifetime; + readonly lifetime?: StackNamespace.StackLifetime; /** * Starts the owner at creation and lets it register the stack under its lease, so a failed or * interrupted launch leaves no registration behind. Session stacks always do this. @@ -328,7 +329,7 @@ const firstUnremovableDirectory = ( * remove the containers and engine-volume data left behind. */ const destroyWithoutEngine = Effect.fn("Stack.destroyWithoutEngine")(function* ( - state: State.Interface, + state: StackNamespace.Interface, saved: SavedStack, locations: StackLocations, engine: "docker" | "podman", @@ -339,13 +340,15 @@ const destroyWithoutEngine = Effect.fn("Stack.destroyWithoutEngine")(function* ( const dataRoot = path.join(locations.stateRoot, id, "data"); return yield* Effect.scoped( Effect.gen(function* () { - if (!(yield* state.lease(id))) - return yield* failure( - "destroy", - "An owner for this stack started during destroy; run destroy again", + const lease = yield* state + .acquireLease(id) + .pipe( + Effect.catchTag("Namespace.LeaseHeldError", () => + failure("destroy", "An owner for this stack started during destroy; run destroy again"), + ), ); - yield* Effect.addFinalizer(() => state.retractHolder(id).pipe(Effect.ignore)); - yield* state.publishHolder(id, { + yield* Effect.addFinalizer(() => lease.retractHolder.pipe(Effect.ignore)); + yield* lease.publishHolder({ role: "sweeper", pid: process.pid, startedAt: DateTime.formatIso(yield* DateTime.now), @@ -383,7 +386,7 @@ const destroyWithoutEngine = Effect.fn("Stack.destroyWithoutEngine")(function* ( type Reach = "launch" | "attach"; const makeHandle = Effect.fn("Stack.makeHandle")(function* ( - state: State.Interface, + state: StackNamespace.Interface, saved: SavedStack, locations: StackLocations, seed: { readonly access?: HostAccess; readonly creator?: boolean } = {}, @@ -973,12 +976,15 @@ export const open = Effect.fn("Stack.open")( export const discover = Effect.fn("Stack.discover")( function* ( options: Pick & { - readonly onInvalidState?: (id: string, error: State.StateError) => Effect.Effect; + readonly onInvalidState?: ( + id: string, + error: StackNamespace.NamespaceError, + ) => Effect.Effect; }, ) { - const state = yield* State.Service.pipe( + const state = yield* StackNamespace.Service.pipe( Effect.provide( - State.layer({ root: options.stateRoot, onInvalidState: options.onInvalidState }), + StackNamespace.layer({ root: options.stateRoot, onInvalidState: options.onInvalidState }), ), ); const saved = yield* state.list; diff --git a/packages/stack/src/host/CommandRunner.initialization.integration.test.ts b/packages/stack/src/host/CommandRunner.initialization.integration.test.ts index 269d9307c2..86195a5867 100644 --- a/packages/stack/src/host/CommandRunner.initialization.integration.test.ts +++ b/packages/stack/src/host/CommandRunner.initialization.integration.test.ts @@ -10,7 +10,7 @@ import { type ArtifactSource, } from "../preparation/ArtifactStore.ts"; import { PreparationError } from "../preparation/Errors.ts"; -import type { StackCredentials } from "../State.ts"; +import type { StackCredentials } from "../StackNamespace.ts"; import * as CommandRunner from "./CommandRunner.ts"; const target = diff --git a/packages/stack/src/host/CommandRunner.ts b/packages/stack/src/host/CommandRunner.ts index 1a318c2892..b75dc0696d 100644 --- a/packages/stack/src/host/CommandRunner.ts +++ b/packages/stack/src/host/CommandRunner.ts @@ -25,7 +25,7 @@ import { makeContainerRuntime, type HostGateway } from "../runtime/Container.ts" import { spawnNativeProcess } from "../runtime/NativeProcess.ts"; import { awaitCommandOutput, type CommandOutputResult } from "../runtime/CommandOutput.ts"; import type { CommandInvocation as CommandInvocationType } from "../Commands.ts"; -import type { StackCredentials } from "../State.ts"; +import type { StackCredentials } from "../StackNamespace.ts"; import { resolveInitializationCommand } from "../services/Initialization.ts"; export class CommandError extends Data.TaggedError("CommandError")<{ diff --git a/packages/stack/src/host/Credentials.ts b/packages/stack/src/host/Credentials.ts index 64705e0452..65e5017e33 100644 --- a/packages/stack/src/host/Credentials.ts +++ b/packages/stack/src/host/Credentials.ts @@ -6,7 +6,7 @@ import { } from "../Defaults.ts"; import { ServiceCreation, type ServiceCreationInput } from "../services/Catalog.ts"; import { resolveStackKeys } from "../services/ServiceConfig.ts"; -import type { SavedStack, StackCredentials, StackKeysInput } from "../State.ts"; +import type { SavedStack, StackCredentials, StackKeysInput } from "../StackNamespace.ts"; export class CredentialError extends Data.TaggedError("CredentialError")<{ readonly message: string; diff --git a/packages/stack/src/index.ts b/packages/stack/src/index.ts index c1df2cbe77..1aa508586b 100644 --- a/packages/stack/src/index.ts +++ b/packages/stack/src/index.ts @@ -12,7 +12,7 @@ export type { PostgresCommandOptions, Stack, } from "./PromiseClient.ts"; -export type { StackCredentials, StackKeysInput } from "./State.ts"; +export type { StackCredentials, StackKeysInput } from "./StackNamespace.ts"; export { initialization, postgres } from "./Commands.ts"; export { StackError } from "./Rpc.ts"; export type { CompositionConfig } from "./Orchestrator.ts"; diff --git a/packages/stack/src/internal/dispatch.integration.test.ts b/packages/stack/src/internal/dispatch.integration.test.ts index 7a547d4ef4..61364b2536 100644 --- a/packages/stack/src/internal/dispatch.integration.test.ts +++ b/packages/stack/src/internal/dispatch.integration.test.ts @@ -4,7 +4,7 @@ import { expect, it } from "@effect/vitest"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { fileURLToPath } from "node:url"; import { HostEndpoint } from "../HostProcess.ts"; -import * as State from "../State.ts"; +import * as StackNamespace from "../StackNamespace.ts"; const fixturePath = fileURLToPath( new URL("../../tests/compiled-dispatch-fixture.ts", import.meta.url), @@ -56,8 +56,8 @@ it.live("dispatches compiled owner and native launchers through the production b yield* compileFixture(executable); const stateRoot = path.join(root, "state"); const cacheRoot = path.join(root, "cache"); - const stateContext = yield* Layer.build(State.layer({ root: stateRoot })); - const state = Context.get(stateContext, State.Service); + const stateContext = yield* Layer.build(StackNamespace.layer({ root: stateRoot })); + const state = Context.get(stateContext, StackNamespace.Service); const stackId = "compiled-dispatch"; yield* state.save({ id: stackId, diff --git a/packages/stack/src/internal/host-process.ts b/packages/stack/src/internal/host-process.ts index eb459d0c37..ddab65508d 100644 --- a/packages/stack/src/internal/host-process.ts +++ b/packages/stack/src/internal/host-process.ts @@ -1,7 +1,7 @@ import { Cause, Effect, Exit, Option, Schema } from "effect"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- readiness is an inherited launcher descriptor. import { closeSync, writeSync } from "node:fs"; -import { SavedStack } from "../State.ts"; +import { SavedStack } from "../StackNamespace.ts"; import { runStackHost, StackHostError, type StackHostOptions } from "../StackHost.ts"; const writeLine = (value: unknown) => diff --git a/packages/stack/src/lifetime.integration.test.ts b/packages/stack/src/lifetime.integration.test.ts index da1a17a058..9998291fbb 100644 --- a/packages/stack/src/lifetime.integration.test.ts +++ b/packages/stack/src/lifetime.integration.test.ts @@ -16,7 +16,7 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; import { create, open } from "./effect.ts"; -import * as State from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; import { assertOwnerExited, captureOwnerPid, watchLeaseRelease } from "../tests/owner.ts"; class LifetimeTestError extends Data.TaggedError("LifetimeTestError")<{ @@ -30,8 +30,8 @@ const sessionFixture = fileURLToPath( ); const stateFor = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), ); const collect = (child: ChildProcessSpawner.ChildProcessHandle) => diff --git a/packages/stack/src/namespace/Capabilities.ts b/packages/stack/src/namespace/Capabilities.ts new file mode 100644 index 0000000000..c464e73614 --- /dev/null +++ b/packages/stack/src/namespace/Capabilities.ts @@ -0,0 +1,39 @@ +import { Duration, Effect, Predicate, Schedule, Schema } from "effect"; + +/** A failure from any namespace operation; `operation` names the step that failed. */ +export class NamespaceError extends Schema.TaggedError()( + "Namespace.NamespaceError", + { + operation: Schema.String, + message: Schema.String, + cause: Schema.optionalKey(Schema.Defect()), + }, +) {} + +export const namespaceError = (operation: string, cause: unknown): NamespaceError => + new NamespaceError({ + operation, + message: cause instanceof Error ? cause.message : String(cause), + cause, + }); + +/** The `code` of a `PlatformError`-shaped `{ cause: { code } }`, or of any error that cause wraps. */ +export const errorCode = (error: unknown): string | undefined => { + if (!Predicate.hasProperty(error, "cause")) return undefined; + return Predicate.hasProperty(error.cause, "code") && typeof error.cause.code === "string" + ? error.cause.code + : undefined; +}; +/** Windows reports a file another process is still using or replacing as a transient violation. */ +export const isSharingViolation = + (platform: NodeJS.Platform) => + (error: unknown): boolean => + platform === "win32" && ["EPERM", "EACCES", "EBUSY"].includes(errorCode(error) ?? ""); + +/** Shared bounded backoff for retrying a transient Windows sharing violation. */ +export const transientRetrySchedule = Schedule.exponential("10 millis", 2).pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.millis(100))), + ), + Schedule.upTo({ times: 12 }), +); diff --git a/packages/stack/src/namespace/Lease.ts b/packages/stack/src/namespace/Lease.ts new file mode 100644 index 0000000000..7afab4a968 --- /dev/null +++ b/packages/stack/src/namespace/Lease.ts @@ -0,0 +1,302 @@ +import { Effect, Exit, FileSystem, Option, Path, Schedule, Schema, Scope, Semaphore } from "effect"; +import type { DatabaseSync } from "node:sqlite"; +import { + isSharingViolation, + namespaceError, + transientRetrySchedule, + type NamespaceError, +} from "./Capabilities.ts"; +import * as Publication from "./Publication.ts"; +import { removeEmptyDirectory } from "./drivers/FileSystem.ts"; +import { acquireLock, errcode, isBusy, takeLock } from "./drivers/Sqlite.ts"; + +/** What the current lease holder publishes: an owner's control endpoint, or a sweeper's marker. */ +export const LeaseHolder = Schema.Union([ + Schema.Struct({ + role: Schema.Literal("owner"), + port: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 65535 })), + pid: Schema.Int, + release: Schema.String, + lifetime: Schema.Literals(["session", "detached"]), + startedAt: Schema.String, + secret: Schema.String, + }), + Schema.Struct({ role: Schema.Literal("sweeper"), pid: Schema.Int, startedAt: Schema.String }), +]); +export type LeaseHolder = Schema.Schema.Type; + +/** A live owner already holds this stack's lease. */ +export class LeaseHeldError extends Schema.TaggedError()( + "Namespace.LeaseHeldError", + { + stackId: Schema.String, + message: Schema.String, + }, +) {} + +/** + * Obtained only by acquiring a stack's owner lease, for the life of that acquisition's scope. + * `publishHolder` and `retractHolder` are the only way to write or clear this process's holder + * record, so neither can run without the lease that makes the record meaningful. + */ +interface LeaseHandle { + readonly stackId: string; + readonly ownerLog: string; + readonly publishHolder: (record: LeaseHolder) => Effect.Effect; + readonly retractHolder: Effect.Effect; +} + +export interface Interface { + readonly acquireLease: ( + id: string, + ) => Effect.Effect; + /** Reports whether any process currently holds the stack's owner lease. */ + readonly leased: (id: string) => Effect.Effect; + /** Only meaningful while the lease is held; a record left by a dead holder is stale. */ + readonly readHolder: (id: string) => Effect.Effect; + /** The file that receives the stack owner's stdout and stderr. */ + readonly ownerLog: (id: string) => string; +} + +const checkId = (id: string): Effect.Effect => + /^[a-zA-Z0-9_-]+$/u.test(id) + ? Effect.void + : Effect.fail(namespaceError("identity", `Invalid state id: ${id}`)); + +export interface Options { + readonly root: string; + readonly platform?: NodeJS.Platform; + /** Whether the registry still has a saved document for this stack id. */ + readonly isRegistered: (id: string) => Effect.Effect; + /** Observes a lease request that found the lease held and is waiting for it. */ + readonly onLeaseContended?: (id: string) => Effect.Effect; +} + +export const make = ( + options: Options, +): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stackRoot = (id: string) => path.join(options.root, id); + const leasePath = (id: string) => path.join(stackRoot(id), "owner.lock"); + const ownerPath = (id: string) => path.join(stackRoot(id), "owner.json"); + const ownerLog = (id: string) => path.join(stackRoot(id), "owner.log"); + const sharingViolation = isSharingViolation(options.platform ?? process.platform); + const retryTransientRead = (effect: Effect.Effect) => + effect.pipe(Effect.retry({ schedule: transientRetrySchedule, while: sharingViolation })); + + const isMissing = (error: NamespaceError) => errcode(error.cause, 14); + /** An open file that was unlinked: SQLite IOERR_VNODE on macOS, IOERR_FSTAT on Linux. */ + const isMoved = (error: NamespaceError) => + errcode(error.cause, 6922) || errcode(error.cause, 1802); + /** Deletes the lease file of an unregistered stack; `false` means it is still in place. */ + const removeLeaseFile = (id: string) => + options.isRegistered(id).pipe( + Effect.flatMap((registered) => + registered ? Effect.void : fs.remove(leasePath(id), { force: true }), + ), + Effect.as(true), + Effect.orElseSucceed(() => false), + ); + const fileIdentity = (file: string) => + fs.stat(file).pipe( + Effect.map((info) => `${info.dev}:${Option.getOrElse(info.ino, () => "")}`), + Effect.option, + ); + + const acquireLease = Effect.fn("Namespace.Lease.acquire")(function* (id: string) { + yield* checkId(id); + const target = leasePath(id); + const scope = yield* Scope.Scope; + let held = false; + let unlinked = false; + let closed = false; + // Shared by the mutations and the finalizer below, so a publish or retract already in + // flight finishes before the finalizer marks the handle closed and releases the connection. + const gate = yield* Semaphore.make(1); + yield* Effect.addFinalizer(() => + held + ? (unlinked ? Effect.void : removeLeaseFile(id)).pipe( + Effect.andThen(removeEmptyDirectory(stackRoot(id))), + Effect.ignore, + ) + : Effect.void, + ); + let contended = false; + const noteContention = Effect.suspend(() => + contended + ? Effect.void + : Effect.sync(() => (contended = true)).pipe( + Effect.andThen(options.onLeaseContended?.(id) ?? Effect.void), + ), + ); + + /** Closes `attemptScope`, noting contention first so busy is reported at most once. */ + const abandon = ( + attemptScope: Scope.Closeable, + error: NamespaceError, + ): Effect.Effect => + (isBusy(error) ? noteContention : Effect.void).pipe( + Effect.andThen(Scope.close(attemptScope, Exit.void)), + Effect.andThen(Effect.fail(error)), + ); + + /** + * A waiter can open the lease file just before its holder unlinks it, and then lock the + * unlinked file; comparing the path's file before opening and after locking rejects that. + * Up to 8 quick retries cover that race; a busy failure propagates instead, so the retry + * around this whole attempt (below) can wait it out on a slower schedule. + */ + const tryOnce: Effect.Effect<{ readonly ownerLog: string }, NamespaceError, Scope.Scope> = + Effect.gen(function* () { + for (let attempt = 0; attempt < 8; attempt++) { + // A releasing holder may remove the empty stack directory at any moment. + yield* fs + .makeDirectory(stackRoot(id), { recursive: true, mode: 0o700 }) + .pipe(Effect.mapError((cause) => namespaceError("lease", cause))); + const before = yield* fileIdentity(target); + const attemptScope = yield* Scope.fork(scope, "sequential"); + const connection = yield* acquireLock(target, "create").pipe( + Scope.provide(attemptScope), + Effect.map(Option.some), + Effect.catchIf(isMissing, () => Effect.succeed(Option.none())), + Effect.catch((error) => abandon(attemptScope, error)), + ); + if (Option.isNone(connection)) { + yield* Scope.close(attemptScope, Exit.void); + continue; + } + const locked = yield* takeLock(connection.value).pipe( + Effect.as(true), + Effect.catchIf(isMoved, () => Effect.succeed(false)), + Effect.catch((error) => abandon(attemptScope, error)), + ); + const after = locked ? yield* fileIdentity(target) : Option.none(); + if (Option.isSome(before) && Option.isSome(after) && before.value === after.value) { + held = true; + yield* Scope.addFinalizer( + attemptScope, + removeLeaseFile(id).pipe(Effect.map((removed) => (unlinked = removed))), + ); + // Registered last so it runs first: taking the gate here waits for a publish or + // retract already in flight, so no mutation can resume after the connection below + // (registered earlier, so it releases after this) is gone. + yield* Scope.addFinalizer( + attemptScope, + gate.withPermit( + Effect.sync(() => { + closed = true; + }), + ), + ); + return { ownerLog: ownerLog(id) }; + } + yield* Scope.close(attemptScope, Exit.void); + } + return yield* namespaceError("lease", `The lease file of stack ${id} keeps changing`); + }); + + const outcome = yield* tryOnce.pipe( + Effect.retry({ + schedule: Schedule.spaced("25 millis").pipe(Schedule.upTo({ duration: "500 millis" })), + while: isBusy, + }), + Effect.catchIf( + isBusy, + () => + new LeaseHeldError({ + stackId: id, + message: `Another owner holds the lease of stack ${id}`, + }), + ), + ); + // A handle plucked out of its scope (for example, stored past the scope that acquired it) + // must not mutate another holder's live record once that scope has closed; checking under + // the same gate the finalizer takes rules out a mutation that is already past this check + // resuming after the finalizer has closed the handle and released the connection. Waiting + // for the gate stays interruptible, but once acquired, `effect` runs uninterruptibly: a + // filesystem callback that is already in flight (for example `fs.rm`'s) has no cancellation + // of its own, so interrupting mid-flight would free the gate while that work is still + // pending, letting a later holder's record publish before this removal finally lands on it. + const guarded = (effect: Effect.Effect) => + gate.withPermit( + Effect.uninterruptible( + Effect.suspend(() => + closed + ? Effect.fail( + namespaceError("lease", `The lease handle for stack ${id} has already closed`), + ) + : effect, + ), + ), + ); + return { + stackId: id, + ownerLog: outcome.ownerLog, + publishHolder: Effect.fn("Namespace.Lease.publishHolder")(function* (record: LeaseHolder) { + const serialized = yield* Schema.encodeEffect(Schema.fromJsonString(LeaseHolder))( + record, + ).pipe(Effect.mapError((cause) => namespaceError("encode", cause))); + yield* guarded( + // A crashed holder's stale record, if any, must not block this fresh one from publishing. + fs.remove(ownerPath(id), { force: true }).pipe( + Effect.mapError((cause) => namespaceError("remove", cause)), + Effect.andThen( + Publication.publish(fs, path, { + target: ownerPath(id), + content: serialized, + mode: "initial", + platform: options.platform, + }), + ), + ), + ); + }), + retractHolder: guarded( + fs + .remove(ownerPath(id), { force: true }) + .pipe(Effect.mapError((cause) => namespaceError("remove", cause))), + ).pipe(Effect.withSpan("Namespace.Lease.retractHolder")), + } satisfies LeaseHandle; + }); + + const leased = Effect.fn("Namespace.Lease.leased")(function* (id: string) { + yield* checkId(id); + return yield* Effect.scoped( + acquireLock(leasePath(id), "existing").pipe( + Effect.flatMap((connection) => takeLock(connection).pipe(Effect.as(false))), + Effect.catchIf(isBusy, () => Effect.succeed(true)), + ), + ).pipe( + // A lease file that is missing, or that its holder unlinked meanwhile, is not held. + Effect.catchIf( + (error) => isMissing(error) || isMoved(error), + () => Effect.succeed(false), + ), + ); + }); + const decodeHolder = Schema.decodeEffect(Schema.fromJsonString(LeaseHolder)); + const readHolder = Effect.fn("Namespace.Lease.readHolder")(function* (id: string) { + yield* checkId(id); + const target = ownerPath(id); + // The holder may retract its record at any moment, so a missing record is not an error. + const text = yield* fs.readFileString(target).pipe( + retryTransientRead, + Effect.map(Option.some), + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed(Option.none()), + ), + Effect.mapError((cause) => namespaceError("read", cause)), + ); + if (Option.isNone(text)) return undefined; + return yield* decodeHolder(text.value).pipe( + Effect.mapError((cause) => + namespaceError("decode", `Unable to decode ${target}: ${cause}`), + ), + ); + }); + + return { acquireLease, leased, readHolder, ownerLog }; + }); diff --git a/packages/stack/src/namespace/Namespace.e2e.test.ts b/packages/stack/src/namespace/Namespace.e2e.test.ts new file mode 100644 index 0000000000..eef6c1d58b --- /dev/null +++ b/packages/stack/src/namespace/Namespace.e2e.test.ts @@ -0,0 +1,306 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { + Context, + Data, + Deferred, + Effect, + Fiber, + FileSystem, + Layer, + Path, + Ref, + Schema, + Stream, +} from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { fileURLToPath } from "node:url"; +import { + connectHost, + launchHost, + ownerExitProbe, + shutdownHost, + waitForOwnerExit, +} from "../HostProcess.ts"; +import * as StackNamespace from "../StackNamespace.ts"; +import { watchLeaseRelease } from "../../tests/owner.ts"; + +class PublishBarrierError extends Data.TaggedError("PublishBarrierError")<{ + readonly message: string; +}> {} +class LockFixtureError extends Data.TaggedError("LockFixtureError")<{ + readonly message: string; +}> {} + +const makeTestState = (root: string) => + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), + ); + +const ownerFixture = fileURLToPath(new URL("../../tests/host-process-fixture.ts", import.meta.url)); +const publishLoopFixture = fileURLToPath( + new URL("../../tests/publish-loop-fixture.ts", import.meta.url), +); +const lockFixture = fileURLToPath( + new URL("../../tests/namespace-lock-fixture.ts", import.meta.url), +); + +const testLayer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); + +const savedStack = (root: string, id: string): StackNamespace.SavedStack => ({ + id, + runtime: "native", + identity: { projectRoot: root, branchContext: "main", stackName: "local" }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + ports: [], +}); + +it.live( + "a new owner acquires a SIGKILLed owner's lease, with no manual cleanup", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-e2e-sigkill-" }); + const state = yield* makeTestState(root); + const id = "stack"; + yield* state.save(savedStack(root, id)); + const options = { stateRoot: root, cacheRoot: root, stackId: id, entrypoint: ownerFixture }; + + const first = yield* launchHost(state, options); + expect(yield* state.leased(id)).toBe(true); + + const released = yield* watchLeaseRelease(root, id); + yield* Effect.sync(() => process.kill(first.endpoint.pid, "SIGKILL")); + yield* released; + expect(yield* state.leased(id)).toBe(false); + + // No file is removed or repaired here; the kernel alone released the dead owner's lease. + const second = yield* Effect.acquireRelease(launchHost(state, options), (access) => + shutdownHost(access, true).pipe(Effect.ignore), + ); + expect(second.endpoint.pid).not.toBe(first.endpoint.pid); + expect(yield* connectHost(state, id)).toEqual(second); + }), + ).pipe(Effect.provide(testLayer)), + 20_000, +); + +it.live( + "killing a writer exactly at the staging-to-publish boundary leaves the prior document intact and lets the next publish recover", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-e2e-publish-" }); + const state = yield* makeTestState(root); + const id = "stack"; + yield* state.save(savedStack(root, id)); + + const writer = yield* spawner.spawn( + ChildProcess.make(process.execPath, [publishLoopFixture, root, id], { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }), + ); + const ready = yield* Deferred.make(); + const stderr = yield* Ref.make(""); + const diagnostics = yield* writer.stderr.pipe( + Stream.decodeText, + Stream.runForEach((chunk) => Ref.update(stderr, (text) => text + chunk)), + Effect.forkScoped, + ); + const output = yield* writer.stdout.pipe( + Stream.decodeText, + Stream.splitLines, + Stream.tap((line) => + line === "about-to-publish" ? Deferred.succeed(ready, undefined) : Effect.void, + ), + Stream.runDrain, + Effect.forkScoped, + ); + const writerFailure = (reason: string) => + Ref.get(stderr).pipe( + Effect.flatMap((text) => + Effect.fail(new PublishBarrierError({ message: `${reason}: ${text}` })), + ), + ); + // The fixture announces this boundary itself, right before the real rename call; killing + // here lands in the staging-to-publish gap on every run, not "most of the time". + yield* Deferred.await(ready).pipe( + Effect.raceFirst( + writer.exitCode.pipe( + Effect.matchEffect({ + onFailure: (cause) => + writerFailure(`Writer exited before readiness: ${String(cause)}`), + onSuccess: (code) => writerFailure(`Writer exited before readiness (${code})`), + }), + ), + ), + Effect.timeoutOrElse({ + duration: "10 seconds", + orElse: () => writerFailure("Writer did not reach the publish boundary"), + }), + ); + yield* Effect.sync(() => process.kill(Number(writer.pid), "SIGKILL")); + yield* writer.exitCode.pipe(Effect.ignore); + yield* Fiber.join(output); + yield* Fiber.join(diagnostics); + + const target = path.join(root, id, "state.json"); + const raw = yield* fs.readFileString(target); + const decoded = yield* Schema.decodeEffect( + Schema.fromJsonString(StackNamespace.SavedStack), + )(raw); + // The killed writer never reached the rename: the prior document is untouched, and only + // its staging file, never a partial target, is left behind. + expect(decoded).toEqual(savedStack(root, id)); + expect( + (yield* fs.readDirectory(path.join(root, id))).filter((entry) => entry.endsWith(".tmp")), + ).toHaveLength(1); + + yield* state.save({ + ...decoded, + identity: { ...decoded.identity, stackName: "recovered" }, + }); + expect(yield* state.read(id)).toEqual({ + ...decoded, + identity: { ...decoded.identity, stackName: "recovered" }, + }); + }), + ).pipe(Effect.provide(testLayer)), + 20_000, +); + +it.live( + "serializes real writer processes and releases a killed registry holder while its child survives", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-e2e-lock-process-" }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const spawn = (args: ReadonlyArray) => + spawner.spawn( + ChildProcess.make(process.execPath, [lockFixture, ...args], { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + detached: true, + forceKillAfter: "1 second", + }), + ); + const readOutput = (child: ChildProcessSpawner.ChildProcessHandle) => + Effect.all( + [ + child.stdout.pipe(Stream.decodeText, Stream.mkString), + child.stderr.pipe(Stream.decodeText, Stream.mkString), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + + const state = yield* makeTestState(root); + const saved = savedStack(root, "stack-main"); + yield* state.save(saved); + + yield* Effect.forEach( + ["first", "second", "third"], + (id) => + Effect.scoped( + Effect.gen(function* () { + const writer = yield* spawn(["write", root, id]); + const [, stderr, code] = yield* readOutput(writer); + expect(code, stderr).toBe(0); + }), + ), + { concurrency: "unbounded" }, + ); + expect((yield* state.read(saved.id))?.instances.map(({ id }) => id).sort()).toEqual([ + "first", + "second", + "third", + ]); + + const holder = yield* spawn(["hold", root]); + const locked = yield* Deferred.make(); + const childReady = yield* Deferred.make(); + const stderr = yield* Ref.make(""); + const diagnostics = yield* holder.stderr.pipe( + Stream.decodeText, + Stream.runForEach((chunk) => Ref.update(stderr, (text) => text + chunk)), + Effect.forkScoped, + ); + const output = yield* holder.stdout.pipe( + Stream.decodeText, + Stream.splitLines, + Stream.runForEach((line) => + line === "locked" + ? Deferred.succeed(locked, undefined).pipe(Effect.asVoid) + : line.startsWith("child:") + ? Deferred.succeed(childReady, line.slice(6)).pipe(Effect.asVoid) + : Effect.void, + ), + Effect.forkScoped, + ); + const holderFailure = (reason: string) => + Ref.get(stderr).pipe( + Effect.flatMap((text) => + Effect.fail(new LockFixtureError({ message: `${reason}: ${text}` })), + ), + ); + yield* Effect.all([Deferred.await(locked), Deferred.await(childReady)]).pipe( + Effect.raceFirst( + holder.exitCode.pipe( + Effect.matchEffect({ + onFailure: (cause) => + holderFailure(`Holder exited before readiness: ${String(cause)}`), + onSuccess: (code) => holderFailure(`Holder exited before readiness (${code})`), + }), + ), + ), + Effect.timeoutOrElse({ + duration: "10 seconds", + orElse: () => holderFailure("Holder readiness timed out"), + }), + ); + const childPid = yield* Schema.decodeEffect(Schema.FiniteFromString)( + yield* Deferred.await(childReady), + ); + const kill = (pid: number) => + Effect.try({ + try: () => process.kill(pid, "SIGKILL"), + catch: (cause) => new LockFixtureError({ message: String(cause) }), + }); + yield* Effect.addFinalizer(() => + kill(childPid).pipe( + Effect.ignore, + Effect.andThen(Fiber.join(output)), + Effect.andThen(Fiber.join(diagnostics)), + Effect.andThen(waitForOwnerExit(childPid, ownerExitProbe(fs)).pipe(Effect.ignore)), + Effect.timeout("10 seconds"), + Effect.orDie, + ), + ); + yield* kill(Number(holder.pid)); + yield* holder.exitCode.pipe(Effect.exit); + yield* state.withLock( + state.save({ ...saved, identity: { ...saved.identity, stackName: "after-crash" } }), + ); + expect((yield* state.read(saved.id))?.identity.stackName).toBe("after-crash"); + yield* Effect.try({ + try: () => process.kill(childPid, 0), + catch: (cause) => + new LockFixtureError({ + message: `Child did not survive the killed holder: ${String(cause)}`, + }), + }); + }), + ).pipe(Effect.provide(testLayer)), + 20_000, +); diff --git a/packages/stack/src/namespace/Namespace.integration.test.ts b/packages/stack/src/namespace/Namespace.integration.test.ts new file mode 100644 index 0000000000..dcd2ddf6c4 --- /dev/null +++ b/packages/stack/src/namespace/Namespace.integration.test.ts @@ -0,0 +1,1093 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { TestClock } from "effect/testing"; +import { + Context, + Deferred, + Effect, + Exit, + Fiber, + FileSystem, + Layer, + Option, + Path, + PlatformError, + Ref, + Result, + Schema, + Scheduler, + Scope, +} from "effect"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- synchronous existence check between manual scheduler steps. +import { existsSync } from "node:fs"; +import { create, discover } from "../effect.ts"; +import { destroyTestStack } from "../../tests/stack-cleanup.ts"; +import { errorCode } from "./Capabilities.ts"; +import * as StackNamespace from "../StackNamespace.ts"; + +const makeTestState = (root: string) => + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), + ); + +const initial: StackNamespace.SavedStack = { + id: "stack-main", + identity: { + projectRoot: "/tmp/project", + branchContext: "main", + stackName: "local", + }, + runtime: "docker", + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + ports: [], +}; + +const run = (effect: Effect.Effect) => + Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); + +const stackLayer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); + +describe("durable registry", () => { + it.live("reopens saved state and serializes concurrent read-modify-write operations", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-registry-" }); + const first = yield* makeTestState(root); + const second = yield* makeTestState(root); + yield* first.save(initial); + + const add = (store: typeof first, id: string) => + store.withLock( + Effect.gen(function* () { + const current = yield* store.read(initial.id); + if (current === undefined) return yield* Effect.die("state disappeared"); + yield* store.save({ + ...current, + instances: [ + ...current.instances, + { id, creation: { service: "mail", config: {} } }, + ], + }); + }), + ); + yield* Effect.all([add(first, "db-one"), add(second, "db-two")], { + concurrency: "unbounded", + }); + + const reopened = yield* (yield* makeTestState(root)).read(initial.id); + expect(reopened?.instances.map((instance) => instance.id).sort()).toEqual([ + "db-one", + "db-two", + ]); + expect(yield* fs.stat(path.join(root, "stack-main", "state.json"))).toBeDefined(); + }), + ), + ); + + it.effect("rejects unsafe paths and malformed documents without changing valid state", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-registry-invalid-" }); + const store = yield* makeTestState(root); + yield* store.save(initial); + const unsafe = yield* store.read("../outside").pipe(Effect.exit); + expect(Exit.isFailure(unsafe)).toBe(true); + + yield* fs.makeDirectory(`${root}/broken`); + yield* fs.writeFileString(`${root}/broken/state.json`, '{"id":42}'); + const malformed = yield* store.read("broken").pipe(Effect.exit); + expect(Exit.isFailure(malformed)).toBe(true); + if (Exit.isFailure(malformed)) { + expect(String(malformed.cause)).toContain("stack broken"); + expect(String(malformed.cause)).toContain("state.json"); + } + const valid = yield* store.read(initial.id); + expect(Schema.is(StackNamespace.SavedStack)(valid)).toBe(true); + }), + ), + ); + + const listingWithReadFailures = (options: { + readonly root: string; + readonly platform: NodeJS.Platform; + readonly code: string; + readonly failures: number; + }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const target = (yield* Path.Path).join(options.root, initial.id, "state.json"); + const remaining = yield* Ref.make(options.failures); + const firstFailure = yield* Deferred.make(); + const reported = yield* Ref.make>([]); + const injectedFs = Layer.succeed(FileSystem.FileSystem, { + ...fs, + readFileString: (file: string, encoding?: string) => + Effect.gen(function* () { + if (file === target && (yield* Ref.get(remaining)) > 0) { + yield* Ref.update(remaining, (count) => count - 1); + yield* Deferred.succeed(firstFailure, undefined); + return yield* PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method: "readFile", + pathOrDescriptor: file, + cause: Object.assign(new Error("injected read failure"), { code: options.code }), + }); + } + return yield* fs.readFileString(file, encoding); + }), + }); + const store = yield* Layer.build( + StackNamespace.layer({ + root: options.root, + platform: options.platform, + onInvalidState: (id) => Ref.update(reported, (ids) => [...ids, id]), + }).pipe(Layer.provide(injectedFs)), + ).pipe(Effect.map((context) => Context.get(context, StackNamespace.Service))); + return { store, firstFailure, reported }; + }); + + it.effect("lists a stack after a transient Windows read failure clears", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "namespace-registry-list-retry-", + }); + yield* (yield* makeTestState(root)).save(initial); + const { store, firstFailure, reported } = yield* listingWithReadFailures({ + root, + platform: "win32", + code: "EBUSY", + failures: 1, + }); + + const listing = yield* store.list.pipe(Effect.forkScoped); + yield* Deferred.await(firstFailure); + yield* TestClock.adjust("10 millis"); + expect((yield* Fiber.join(listing)).map(({ id }) => id)).toEqual([initial.id]); + expect(yield* Ref.get(reported)).toEqual([]); + }), + ), + ); + + it.effect("skips and reports a stack whose state stays unreadable after retries", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + for (const [platform, code] of [ + ["win32", "EBUSY"], + ["linux", "EACCES"], + ] as const) { + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "namespace-registry-list-skip-", + }); + yield* (yield* makeTestState(root)).save(initial); + const { store, firstFailure, reported } = yield* listingWithReadFailures({ + root, + platform, + code, + failures: Number.POSITIVE_INFINITY, + }); + + const listing = yield* store.list.pipe(Effect.forkScoped); + yield* Deferred.await(firstFailure); + yield* TestClock.adjust("950 millis"); + expect(yield* Fiber.join(listing)).toEqual([]); + expect(yield* Ref.get(reported)).toEqual([initial.id]); + } + }), + ), + ); + + it.live.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "lists and claims readable stacks past a sibling directory it cannot access", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-registry-eacces-" }); + const reported: Array = []; + const store = yield* Layer.build( + StackNamespace.layer({ + root, + onInvalidState: (id) => Effect.sync(() => reported.push(id)), + }), + ).pipe(Effect.map((context) => Context.get(context, StackNamespace.Service))); + yield* store.save(initial); + const locked = path.join(root, "locked"); + yield* fs.makeDirectory(locked, { mode: 0o700 }); + yield* fs.writeFileString(path.join(locked, "state.json"), "{}"); + yield* Effect.acquireRelease(fs.chmod(locked, 0o000), () => + fs.chmod(locked, 0o700).pipe(Effect.orDie), + ); + + expect((yield* store.list).map(({ id }) => id)).toEqual([initial.id]); + expect(reported).toEqual(["locked"]); + expect((yield* store.claims).map(({ id }) => id)).toEqual([initial.id]); + }), + ), + ); + + it.live("skips and reports malformed, mismatched, and non-file entries while listing", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "namespace-registry-list-invalid-", + }); + const reported: Array = []; + const store = yield* Layer.build( + StackNamespace.layer({ + root, + onInvalidState: (id) => Effect.sync(() => reported.push(id)), + }), + ).pipe(Effect.map((context) => Context.get(context, StackNamespace.Service))); + yield* store.save(initial); + yield* fs.makeDirectory(path.join(root, "malformed")); + yield* fs.writeFileString(path.join(root, "malformed", "state.json"), "{broken"); + yield* fs.makeDirectory(path.join(root, "mismatched")); + yield* fs.writeFileString( + path.join(root, "mismatched", "state.json"), + yield* Schema.encodeEffect(Schema.fromJsonString(StackNamespace.SavedStack))(initial), + ); + yield* fs.makeDirectory(path.join(root, "directory", "state.json"), { recursive: true }); + yield* fs.writeFileString(path.join(root, "stray-file"), ""); + + expect((yield* store.list).map(({ id }) => id)).toEqual([initial.id]); + // Windows resolves a path below a regular file as missing rather than unreadable. + expect(reported.toSorted()).toEqual( + process.platform === "win32" + ? ["directory", "malformed", "mismatched"] + : ["directory", "malformed", "mismatched", "stray-file"], + ); + }), + ), + ); + + it.live("decodes saved compositions, creations, and instance ids strictly", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-registry-typed-" }); + const reported: Array = []; + const store = yield* Layer.build( + StackNamespace.layer({ + root, + onInvalidState: (id) => Effect.sync(() => reported.push(id)), + }), + ).pipe(Effect.map((context) => Context.get(context, StackNamespace.Service))); + yield* store.save(initial); + const mail = { service: "mail", config: {} }; + const invalid = { + "bad-composition": { composition: { members: "none", dependencies: [] } }, + "bad-creation": { instances: [{ id: "one", creation: { service: "unknown" } }] }, + "duplicate-ids": { + instances: [ + { id: "one", creation: mail }, + { id: "one", creation: mail }, + ], + }, + }; + for (const [id, override] of Object.entries(invalid)) { + yield* fs.makeDirectory(path.join(root, id)); + yield* fs.writeFileString( + path.join(root, id, "state.json"), + yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))({ + ...initial, + id, + ...override, + }), + ); + } + + expect((yield* store.list).map(({ id }) => id)).toEqual([initial.id]); + expect(reported.toSorted()).toEqual(Object.keys(invalid)); + for (const id of Object.keys(invalid)) { + const failure = yield* store.read(id).pipe(Effect.flip); + expect(failure.operation).toBe("decode"); + expect(failure.message).toContain("Unable to decode state"); + expect(failure.message).toContain(`for stack ${id}`); + } + const duplicate = yield* store.read("duplicate-ids").pipe(Effect.flip); + expect(duplicate.message).toContain("Expected unique instance ids"); + }), + ), + ); + + it.live("removes empty stack parents without deleting unowned data", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-registry-remove-" }); + const store = yield* makeTestState(root); + yield* store.save(initial); + yield* fs.makeDirectory(path.join(root, initial.id, "data"), { recursive: true }); + yield* store.remove(initial.id); + expect(yield* fs.exists(path.join(root, initial.id))).toBe(false); + + yield* store.save(initial); + yield* fs.makeDirectory(path.join(root, initial.id, "data"), { recursive: true }); + yield* fs.writeFileString(path.join(root, initial.id, "data", "owned-by-caller"), "keep"); + yield* store.remove(initial.id); + expect(yield* fs.exists(path.join(root, initial.id, "data", "owned-by-caller"))).toBe(true); + }), + ), + ); + + it.live.skipIf(process.platform === "win32")( + "restricts the state root to its owner while keeping a traverse-only grant", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "namespace-registry-traverse-", + }); + yield* fs.chmod(root, 0o755); + yield* makeTestState(root); + expect((yield* fs.stat(root)).mode & 0o777).toBe(0o701); + }), + ), + ); +}); + +describe("registry lock", () => { + it.effect("keeps a lock owner protected when a waiting fiber is cancelled", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lock-cancel-" }); + const first = yield* makeTestState(root); + const second = yield* makeTestState(root); + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const owner = yield* Effect.forkScoped( + first.withLock( + Effect.gen(function* () { + yield* Deferred.succeed(entered, undefined); + yield* Deferred.await(release); + }), + ), + ); + yield* Deferred.await(entered); + const waiter = yield* Effect.forkScoped(second.withLock(second.save(initial))); + yield* TestClock.adjust("100 millis"); + yield* Fiber.interrupt(waiter); + expect(yield* second.read(initial.id)).toBeUndefined(); + const later = yield* Effect.forkScoped(second.withLock(second.save(initial))); + yield* TestClock.adjust("100 millis"); + expect(yield* second.read(initial.id)).toBeUndefined(); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(owner); + yield* TestClock.adjust("50 millis"); + yield* Fiber.join(later); + expect(yield* second.read(initial.id)).toEqual(initial); + }), + ), + ); + + it.effect("times out contention without releasing the holder's lock", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lock-timeout-" }); + const first = yield* makeTestState(root); + const second = yield* makeTestState(root); + const entered = yield* Deferred.make(); + const owner = yield* first + .withLock(Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never))) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + const waiter = yield* second + .withLock(second.save(initial)) + .pipe(Effect.flip, Effect.forkScoped); + yield* TestClock.adjust("6 seconds"); + const result = yield* Fiber.join(waiter); + expect(result.operation).toBe("lock"); + expect(result.message).toBe("Stack registry is locked by another operation; retry shortly"); + expect(yield* second.read(initial.id)).toBeUndefined(); + yield* Fiber.interrupt(owner); + yield* second.withLock(second.save(initial)); + expect(yield* second.read(initial.id)).toEqual(initial); + }), + ), + ); + + it.live("releases ownership after failure and defects without replacing the lock file", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lock-failure-" }); + const state = yield* makeTestState(root); + const failure = new StackNamespace.NamespaceError({ + operation: "test", + message: "body failed", + }); + expect(yield* state.withLock(Effect.fail(failure)).pipe(Effect.flip)).toBe(failure); + const defect = yield* state.withLock(Effect.die("body defect")).pipe(Effect.exit); + expect(Exit.hasDies(defect)).toBe(true); + yield* state.withLock(state.save(initial)); + expect(yield* state.read(initial.id)).toEqual(initial); + expect((yield* fs.readDirectory(root)).sort()).toEqual([ + ".registry-lock.sqlite", + initial.id, + ]); + expect((yield* fs.stat(`${root}/.registry-lock.sqlite`)).size).toBe(0n); + }), + ), + ); + + it.effect("fails invalid lock files without retrying or changing saved state", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lock-invalid-" }); + const state = yield* makeTestState(root); + yield* fs.writeFileString(`${root}/.registry-lock.sqlite`, "not a database"); + const error = yield* state.withLock(state.save(initial)).pipe(Effect.flip); + expect(error).toBeInstanceOf(StackNamespace.NamespaceError); + expect(error.operation).toBe("lock"); + expect(yield* state.read(initial.id)).toBeUndefined(); + }), + ), + ); + + it.live( + "releases the lock when the holder is interrupted right after taking it, before the caller body runs", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lock-interrupt-" }); + const first = yield* makeTestState(root); + const second = yield* makeTestState(root); + const lockPath = `${root}/.registry-lock.sqlite`; + const started = yield* Deferred.make(); + + // Yielding after every single fiber step, and only resuming a step once flushed, gives + // deterministic single-step control over the forked fiber below, with no production seam. + const tasks: Array<() => void> = []; + const dispatcher: Scheduler.SchedulerDispatcher = { + scheduleTask: (task) => tasks.push(task), + flush: () => { + for (const task of tasks.splice(0, tasks.length)) task(); + }, + }; + const stepScheduler: Scheduler.Scheduler = { + executionMode: "async", + shouldYield: (fiber) => fiber.currentOpCount >= fiber.maxOpsBeforeYield, + makeDispatcher: () => dispatcher, + }; + + const holder = yield* first + .withLock(Deferred.succeed(started, undefined).pipe(Effect.andThen(Effect.never))) + .pipe( + Effect.provideService(Scheduler.Scheduler, stepScheduler), + Effect.provideService(Scheduler.MaxOpsBeforeYield, 10), + Effect.forkScoped, + ); + // The fork itself is scheduled on the default scheduler; let it actually start (handing + // control to `stepScheduler`) before single-stepping it below. + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + // Single-step the acquisition until the lock file exists on disk, then stop: this always + // lands in the gap between "lock taken" and "the caller body starts" that the fix closes. + for (let step = 0; step < 20_000 && !existsSync(lockPath); step++) dispatcher.flush(); + expect(existsSync(lockPath)).toBe(true); + expect(holder.pollUnsafe()).toBeUndefined(); + expect(Option.isNone(yield* Deferred.poll(started))).toBe(true); + + // Interrupting and resuming through the same controlled dispatcher keeps the whole + // sequence deterministic: no real scheduler or timer is involved on either side. + holder.interruptUnsafe(); + for (let step = 0; step < 20_000 && holder.pollUnsafe() === undefined; step++) + dispatcher.flush(); + expect(holder.pollUnsafe()).not.toBeUndefined(); + + // No contention wait: a leaked lock would make this block for up to 5 seconds. + yield* second.withLock(second.save(initial)).pipe(Effect.timeout("200 millis")); + expect(yield* second.read(initial.id)).toEqual(initial); + }), + ), + ); +}); + +describe("stack owner lease", () => { + it.live("hands the lease of a removed stack to a waiter on a live lease file", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lease-handoff-" }); + const holder = yield* makeTestState(root); + const contended = yield* Deferred.make(); + const waiter = Context.get( + yield* Layer.build( + StackNamespace.layer({ + root, + onLeaseContended: () => Deferred.succeed(contended, undefined).pipe(Effect.asVoid), + }), + ), + StackNamespace.Service, + ); + const observer = yield* makeTestState(root); + const holderScope = yield* Scope.make(); + const holderLease = yield* holder.acquireLease("gone").pipe(Scope.provide(holderScope)); + expect(holderLease.stackId).toBe("gone"); + + const waiterScope = yield* Scope.make(); + const waiting = yield* waiter + .acquireLease("gone") + .pipe(Scope.provide(waiterScope), Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(contended); + yield* Scope.close(holderScope, Exit.void); + + expect((yield* Fiber.join(waiting)).stackId).toBe("gone"); + expect(yield* observer.leased("gone"), "the path names the file the waiter locked").toBe( + true, + ); + yield* Scope.close(waiterScope, Exit.void); + expect(yield* observer.leased("gone")).toBe(false); + expect(yield* fs.exists(`${root}/gone`), "the last holder removes the directory").toBe( + false, + ); + }), + ), + ); + + it.live("fails a second owner with a typed error while the first still holds the lease", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lease-held-" }); + const holder = yield* makeTestState(root); + const contender = yield* makeTestState(root); + const holderScope = yield* Scope.make(); + yield* holder.acquireLease("busy").pipe(Scope.provide(holderScope)); + + const rejection = yield* contender.acquireLease("busy").pipe(Effect.scoped, Effect.flip); + expect(Schema.is(StackNamespace.LeaseHeldError)(rejection)).toBe(true); + if (!Schema.is(StackNamespace.LeaseHeldError)(rejection)) + return yield* Effect.die(rejection); + expect(rejection.stackId).toBe("busy"); + + yield* Scope.close(holderScope, Exit.void); + }), + ), + ); + + it.live("rejects mutations from a handle whose scope already closed", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lease-expired-" }); + const state = yield* makeTestState(root); + const firstScope = yield* Scope.make(); + const expired = yield* state.acquireLease("recycled").pipe(Scope.provide(firstScope)); + yield* Scope.close(firstScope, Exit.void); + + const secondScope = yield* Scope.make(); + const live = yield* state.acquireLease("recycled").pipe(Scope.provide(secondScope)); + yield* live.publishHolder({ + role: "sweeper", + pid: process.pid, + startedAt: "2026-01-01T00:00:00.000Z", + }); + + const publishRejected = yield* expired + .publishHolder({ + role: "sweeper", + pid: process.pid, + startedAt: "2026-01-01T00:00:00.000Z", + }) + .pipe(Effect.flip); + expect(publishRejected).toBeInstanceOf(StackNamespace.NamespaceError); + const retractRejected = yield* expired.retractHolder.pipe(Effect.flip); + expect(retractRejected).toBeInstanceOf(StackNamespace.NamespaceError); + + expect(yield* state.readHolder("recycled")).toEqual({ + role: "sweeper", + pid: process.pid, + startedAt: "2026-01-01T00:00:00.000Z", + }); + yield* Scope.close(secondScope, Exit.void); + }), + ), + ); + + it.live( + "keeps a fresh holder's record intact against a publish already in flight when its scope closes", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const id = "racing"; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lease-race-" }); + const ownerJson = path.join(root, id, "owner.json"); + // A's publish pauses here, already past the closed-handle guard, so the scope-closing + // finalizer below can only proceed once A's publish has fully released the gate. + const paused = yield* Deferred.make(); + const entered = yield* Deferred.make(); + const removeCalls = yield* Ref.make(0); + const pausingFs = Layer.effect( + FileSystem.FileSystem, + Effect.sync(() => ({ + ...fs, + remove: (file: string, options?: Parameters[1]) => + Effect.gen(function* () { + const isFirst = + file === ownerJson && + (yield* Ref.getAndUpdate(removeCalls, (n) => n + 1)) === 0; + if (isFirst) { + yield* Deferred.succeed(entered, undefined); + yield* Deferred.await(paused); + } + return yield* fs.remove(file, options); + }), + })), + ); + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root }).pipe(Layer.provide(pausingFs))), + StackNamespace.Service, + ); + + const scopeA = yield* Scope.make(); + const a = yield* state.acquireLease(id).pipe(Scope.provide(scopeA)); + const recordA: StackNamespace.LeaseHolder = { + role: "sweeper", + pid: process.pid, + startedAt: "2026-01-01T00:00:00.000Z", + }; + const publishingA = yield* a.publishHolder(recordA).pipe(Effect.forkScoped); + yield* Deferred.await(entered); + + const closingA = yield* Scope.close(scopeA, Exit.void).pipe(Effect.forkScoped); + const recordB: StackNamespace.LeaseHolder = { + role: "sweeper", + pid: process.pid + 1, + startedAt: "2026-01-01T00:00:01.000Z", + }; + const acquiringB = yield* Effect.scoped( + Effect.gen(function* () { + const b = yield* state.acquireLease(id); + yield* b.publishHolder(recordB); + }), + ).pipe(Effect.forkScoped); + + // Neither A's close nor B's acquisition can complete while A's publish is paused. + yield* Effect.sleep("50 millis"); + expect(closingA.pollUnsafe()).toBeUndefined(); + expect(acquiringB.pollUnsafe()).toBeUndefined(); + + yield* Deferred.succeed(paused, undefined); + yield* Fiber.join(publishingA); + yield* Fiber.join(closingA); + yield* Fiber.join(acquiringB); + + expect(yield* state.readHolder(id)).toEqual(recordB); + }), + ), + ); + + it.live( + "keeps a fresh holder's record intact against an interrupted removal whose native callback is still pending", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const id = "racing-interrupt"; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "namespace-lease-race-interrupt-", + }); + const ownerJson = path.join(root, id, "owner.json"); + const entered = yield* Deferred.make(); + const fire = yield* Deferred.make(); + // Settles once the background removal below has actually run, independent of whether + // anything is still listening for its `resume` call: proves the real filesystem effect, + // not just the Effect-level interrupt, has finished before the test checks final state. + const settled = yield* Deferred.make(); + // Only A's own removal (the first call) is held pending; B's later, unrelated calls on + // the same path (its own stale-record cleanup, then its own eventual retraction) must + // pass straight through, or they would deadlock on the very same native callback. + let intercepted = false; + const context = yield* Effect.context(); + const runFork = Effect.runForkWith(context); + // Mirrors `Effect.effectify(fs.rm)`: no cleanup effect is returned, so interrupting the + // fiber waiting on this callback only stops listening for it. The real removal, once + // started in the background below, keeps running and completes on its own regardless. + const callbackFs = Layer.succeed(FileSystem.FileSystem, { + ...fs, + remove: (file: string, options?: Parameters[1]) => + file === ownerJson && !intercepted + ? Effect.callback((resume) => { + intercepted = true; + Deferred.doneUnsafe(entered, Exit.void); + runFork( + Deferred.await(fire).pipe( + Effect.andThen(fs.remove(file, options)), + Effect.matchEffect({ + onSuccess: () => Effect.sync(() => resume(Effect.void)), + onFailure: (cause) => Effect.sync(() => resume(Effect.fail(cause))), + }), + Effect.ensuring(Effect.sync(() => Deferred.doneUnsafe(settled, Exit.void))), + ), + ); + }) + : fs.remove(file, options), + }); + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root }).pipe(Layer.provide(callbackFs))), + StackNamespace.Service, + ); + + const scopeA = yield* Effect.acquireRelease(Scope.make(), (s) => + Scope.close(s, Exit.void), + ); + const a = yield* state.acquireLease(id).pipe(Scope.provide(scopeA)); + + // Runs this one fiber with scheduler yields prevented, so it executes synchronously up + // to the intercepted callback above: by the time this call returns, `entered` is + // already done, which is what makes the steps below a deterministic barrier instead of + // a timing guess. + const mutationContext = Context.add(context, Scheduler.PreventSchedulerYield, true); + const retracting = Effect.runForkWith(mutationContext)(a.retractHolder); + yield* Deferred.await(entered); + yield* Effect.acquireUseRelease( + Effect.void, + () => + Effect.sync(() => { + retracting.interruptUnsafe(); + // Without `Effect.uninterruptible` around the guarded mutation, this interrupt + // unwinds synchronously right here, releasing the gate before the native removal + // below ever runs; with it, the fiber stays suspended until the release phase + // lets the callback fire. + expect(retracting.pollUnsafe()).toBeUndefined(); + }), + () => + Deferred.succeed(fire, undefined).pipe( + Effect.andThen(Fiber.await(retracting)), + Effect.andThen(Deferred.await(settled)), + ), + ); + yield* Scope.close(scopeA, Exit.void); + + const recordB: StackNamespace.LeaseHolder = { + role: "sweeper", + pid: process.pid + 1, + startedAt: "2026-01-01T00:00:01.000Z", + }; + const b = yield* state.acquireLease(id); + yield* b.publishHolder(recordB); + + expect(yield* state.readHolder(id)).toEqual(recordB); + }), + ), + ); + + it.live("reports a free lease without creating a lease file", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lease-probe-" }); + const state = yield* makeTestState(root); + yield* state.save(initial); + expect(yield* state.leased(initial.id)).toBe(false); + expect(yield* state.readHolder(initial.id), "a retracted record reads as absent").toBe( + undefined, + ); + expect(yield* fs.exists(`${root}/${initial.id}/owner.lock`)).toBe(false); + }), + ), + ); + + it.effect("retries a transient holder-read failure and then succeeds", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-holder-read-retry-" }); + const target = path.join(root, initial.id, "owner.json"); + const firstFailure = yield* Deferred.make(); + const failed = yield* Ref.make(false); + const flakyFs = Layer.succeed(FileSystem.FileSystem, { + ...fs, + readFileString: (file: string, encoding?: string) => + Effect.gen(function* () { + if (file === target && !(yield* Ref.get(failed))) { + yield* Ref.set(failed, true); + yield* Deferred.succeed(firstFailure, undefined); + return yield* PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method: "readFile", + pathOrDescriptor: file, + cause: Object.assign(new Error("injected transient read failure"), { + code: "EBUSY", + }), + }); + } + return yield* fs.readFileString(file, encoding); + }), + }); + const state = Context.get( + yield* Layer.build( + StackNamespace.layer({ root, platform: "win32" }).pipe(Layer.provide(flakyFs)), + ), + StackNamespace.Service, + ); + yield* state.save(initial); + const scope = yield* Scope.make(); + const lease = yield* state.acquireLease(initial.id).pipe(Scope.provide(scope)); + const record: StackNamespace.LeaseHolder = { + role: "sweeper", + pid: process.pid, + startedAt: "2026-01-01T00:00:00.000Z", + }; + yield* lease.publishHolder(record); + + const reading = yield* state.readHolder(initial.id).pipe(Effect.forkScoped); + yield* Deferred.await(firstFailure); + yield* TestClock.adjust("10 millis"); + expect(yield* Fiber.join(reading)).toEqual(record); + yield* Scope.close(scope, Exit.void); + }), + ), + ); + + it.live( + "publishes successfully and cleans up staging despite a transient staging-removal failure", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const id = "flaky-cleanup"; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-publish-cleanup-" }); + const stackDirectory = path.join(root, id); + // A single ignored attempt (the pre-fix behavior) only ever makes one call per site; + // three consecutive failures need a real retry loop to still converge. + const remaining = yield* Ref.make(3); + const flakyFs = Layer.succeed(FileSystem.FileSystem, { + ...fs, + remove: (file: string, options?: Parameters[1]) => + Effect.gen(function* () { + if ( + file.startsWith(path.join(stackDirectory, ".owner.json.")) && + (yield* Ref.get(remaining)) > 0 + ) { + yield* Ref.update(remaining, (count) => count - 1); + return yield* PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method: "remove", + pathOrDescriptor: file, + cause: Object.assign(new Error("injected transient removal failure"), { + code: "EBUSY", + }), + }); + } + return yield* fs.remove(file, options); + }), + }); + const state = Context.get( + yield* Layer.build( + StackNamespace.layer({ root, platform: "win32" }).pipe(Layer.provide(flakyFs)), + ), + StackNamespace.Service, + ); + const scope = yield* Scope.make(); + const lease = yield* state.acquireLease(id).pipe(Scope.provide(scope)); + const record: StackNamespace.LeaseHolder = { + role: "sweeper", + pid: process.pid, + startedAt: "2026-01-01T00:00:00.000Z", + }; + yield* lease.publishHolder(record); + + expect(yield* Ref.get(remaining), "all injected failures were consumed").toBe(0); + expect(yield* state.readHolder(id)).toEqual(record); + expect( + (yield* fs.readDirectory(stackDirectory)).filter((entry) => entry.endsWith(".tmp")), + ).toEqual([]); + yield* Scope.close(scope, Exit.void); + }), + ), + ); +}); + +describe("discovery", () => { + it.live("lists identical stack identities under two state roots independently", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-discover-" }); + const projectRoot = `${base}/project`; + yield* fs.makeDirectory(projectRoot, { recursive: true }); + const locations = (root: string) => + ({ + projectRoot, + stateRoot: `${base}/${root}/state`, + cacheRoot: `${base}/${root}/cache`, + runtime: "native", + }) satisfies Parameters[0]; + const a = yield* create(locations("a")); + const b = yield* create(locations("b")); + expect(a.id).toBe(b.id); + + const listedA = yield* discover({ stateRoot: `${base}/a/state` }); + const listedB = yield* discover({ stateRoot: `${base}/b/state` }); + expect(listedA.map(({ definition }) => definition.id)).toEqual([a.id]); + expect(listedB.map(({ definition }) => definition.id)).toEqual([b.id]); + + yield* destroyTestStack(a); + yield* destroyTestStack(b); + }).pipe(Effect.scoped, Effect.provide(stackLayer)), + ); +}); + +describe("publication", () => { + it.effect("rejects a state root without hard-link support before any mutation", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-nolink-" }); + const noLink = Layer.succeed(FileSystem.FileSystem, { + ...fs, + link: (_fromPath: string, toPath: string) => + Effect.fail( + PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method: "link", + pathOrDescriptor: toPath, + description: "Hard links are not supported on this filesystem", + cause: Object.assign(new Error("link not supported"), { code: "ENOTSUP" }), + }), + ), + }); + const failure = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root }).pipe(Layer.provide(noLink))), + Effect.flip, + ); + expect(failure).toBeInstanceOf(StackNamespace.NamespaceError); + expect(failure.operation).toBe("hardlink-support"); + expect(yield* fs.readDirectory(root)).toEqual([]); + }), + ), + ); + + it.live("accepts a root that supports hard links under concurrent acquisitions", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-concurrent-acquire-" }); + const results = yield* Effect.all( + Array.from({ length: 10 }, () => makeTestState(root).pipe(Effect.result)), + { concurrency: "unbounded" }, + ); + expect(results.every(Result.isSuccess)).toBe(true); + }), + ), + ); + + it.effect("fails publication when the directory fsync reports EIO", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const eio = (method: string, file: string) => + PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method, + pathOrDescriptor: file, + cause: Object.assign(new Error("injected directory fsync failure"), { code: "EIO" }), + }); + // Covers both ways opening the directory for its own fsync can report EIO: the `open` + // call itself failing, and `open` succeeding but the handle's own `sync` failing. The + // second case is backed by a real regular-file handle (scoped like any other), not the + // real directory: opening a directory for `sync` hits Windows's own supported `EISDIR` + // path instead of ever reaching the injected failure. + for (const mode of ["open", "sync"] as const) { + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-fsync-eio-" }); + const directory = path.join(root, initial.id); + const syncProbe = path.join(root, ".sync-probe"); + yield* fs.writeFileString(syncProbe, ""); + const failingFs = Layer.succeed(FileSystem.FileSystem, { + ...fs, + open: (file: string, options?: Parameters[1]) => { + if (file !== directory || options?.flag !== "r") return fs.open(file, options); + if (mode === "open") return Effect.fail(eio("open", file)); + return fs + .open(syncProbe, options) + .pipe( + Effect.map((handle) => ({ ...handle, sync: Effect.fail(eio("fsync", file)) })), + ); + }, + }); + const store = yield* Layer.build( + StackNamespace.layer({ root }).pipe(Layer.provide(failingFs)), + ).pipe(Effect.map((context) => Context.get(context, StackNamespace.Service))); + const failure = yield* store.save(initial).pipe(Effect.flip); + expect(failure, mode).toBeInstanceOf(StackNamespace.NamespaceError); + expect(failure.operation, mode).toBe("publish"); + expect(errorCode(failure.cause), mode).toBe("EIO"); + } + }), + ), + ); + + it.effect("leaves no staging file after a partial write fails with ENOSPC", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-publish-enospc-" }); + const directory = path.join(root, initial.id); + const stagingPrefix = path.join(directory, ".state.json."); + const failingFs = Layer.succeed(FileSystem.FileSystem, { + ...fs, + // A real disk ENOSPC lands after some bytes already reached the file; writing a + // truncated prefix for real before failing reproduces that instead of never creating it. + writeFileString: (file: string, content: string, options?: { readonly mode?: number }) => + file.startsWith(stagingPrefix) + ? fs.writeFileString(file, content.slice(0, 1), options).pipe( + Effect.andThen( + Effect.fail( + PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method: "writeFile", + pathOrDescriptor: file, + cause: Object.assign(new Error("injected partial write failure"), { + code: "ENOSPC", + }), + }), + ), + ), + ) + : fs.writeFileString(file, content, options), + }); + const store = yield* Layer.build( + StackNamespace.layer({ root }).pipe(Layer.provide(failingFs)), + ).pipe(Effect.map((context) => Context.get(context, StackNamespace.Service))); + const failure = yield* store.save(initial).pipe(Effect.flip); + expect(failure).toBeInstanceOf(StackNamespace.NamespaceError); + expect( + (yield* fs.readDirectory(directory)).filter((entry) => entry.endsWith(".tmp")), + ).toEqual([]); + }), + ), + ); +}); diff --git a/packages/stack/src/State.windows.integration.test.ts b/packages/stack/src/namespace/Namespace.windows.e2e.test.ts similarity index 90% rename from packages/stack/src/State.windows.integration.test.ts rename to packages/stack/src/namespace/Namespace.windows.e2e.test.ts index cc43a6ca78..649d60c02a 100644 --- a/packages/stack/src/State.windows.integration.test.ts +++ b/packages/stack/src/namespace/Namespace.windows.e2e.test.ts @@ -13,9 +13,9 @@ import { Stream, } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import * as State from "./State.ts"; +import * as StackNamespace from "../StackNamespace.ts"; -const saved: State.SavedStack = { +const saved: StackNamespace.SavedStack = { id: "stack-main", identity: { projectRoot: "C:\\project", branchContext: "test", stackName: "windows" }, runtime: "native", @@ -25,7 +25,7 @@ const saved: State.SavedStack = { ports: [], }; -class HolderError extends Data.TaggedError("StateWindowsHolderError")<{ +class HolderError extends Data.TaggedError("NamespaceWindowsHolderError")<{ readonly message: string; }> {} @@ -42,7 +42,7 @@ const testSharingViolation = () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-windows-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-windows-" }); const path = yield* Path.Path; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const target = path.join(root, saved.id, "state.json"); @@ -74,8 +74,10 @@ const testSharingViolation = () => ), }), ); - const context = yield* Layer.build(State.layer({ root }).pipe(Layer.provide(injectedFs))); - const state = Context.get(context, State.Service); + const context = yield* Layer.build( + StackNamespace.layer({ root }).pipe(Layer.provide(injectedFs)), + ); + const state = Context.get(context, StackNamespace.Service); yield* state.save(saved); yield* fs.writeFileString(path.join(root, "hold-state.ps1"), sharingHolder); @@ -167,9 +169,12 @@ const testSharingViolation = () => ); expect((yield* state.read(saved.id))?.identity.stackName).toBe("recovered"); expect(yield* Ref.get(failures).pipe(Effect.map((seen) => seen.length))).toBeGreaterThan(0); + // Only the staging document the recovered publish produced is left under the stack directory. expect( - (yield* fs.readDirectory(root)).some((entry) => entry.startsWith(".state-write-")), - ).toBe(false); + (yield* fs.readDirectory(path.join(root, saved.id))).filter((entry) => + entry.endsWith(".tmp"), + ), + ).toEqual([]); yield* Fiber.join(output); yield* Fiber.join(diagnostics); }), diff --git a/packages/stack/src/namespace/Publication.ts b/packages/stack/src/namespace/Publication.ts new file mode 100644 index 0000000000..e990b3334e --- /dev/null +++ b/packages/stack/src/namespace/Publication.ts @@ -0,0 +1,58 @@ +import { Effect, FileSystem, Path } from "effect"; +import { isSharingViolation, namespaceError, transientRetrySchedule } from "./Capabilities.ts"; +import * as Drivers from "./drivers/FileSystem.ts"; + +export interface PublishOptions { + readonly target: string; + readonly content: string; + /** "initial" publishes exclusively with a hard link; "replace" atomically renames over it. */ + readonly mode: "initial" | "replace"; + readonly platform?: NodeJS.Platform; +} + +let counter = 0; + +/** + * Writes `content` as `target`, so a reader only ever sees the prior document or the complete new + * one. Stages the full content in `target`'s own directory and fsyncs it, publishes it with a + * hard link or a rename depending on `mode`, then fsyncs the directory. The staging path and its + * cleanup are reserved before anything is written to it, so a write that fails partway (for + * example `ENOSPC`) still leaves no file behind; a process killed mid publish leaves only the + * staging file, never a partially written target. + */ +export const publish = Effect.fn("Namespace.publish")(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + options: PublishOptions, +) { + const directory = path.dirname(options.target); + const staging = path.join( + directory, + `.${path.basename(options.target)}.${process.pid}-${++counter}.tmp`, + ); + const platform = options.platform ?? process.platform; + const retry = (effect: Effect.Effect) => + effect.pipe( + Effect.retry({ schedule: transientRetrySchedule, while: isSharingViolation(platform) }), + ); + yield* Effect.acquireUseRelease( + Effect.succeed(staging), + () => + Effect.gen(function* () { + yield* fs.writeFileString(staging, options.content, { mode: 0o600 }); + yield* Drivers.fsyncFile(fs, staging); + if (options.mode === "initial") { + // Retried separately: once the link lands, the target exists, so retrying it together + // with removing the staging name would see a transient removal failure as `EEXIST` on + // the next attempt instead of as the sharing violation it actually was. + yield* retry(fs.link(staging, options.target)); + yield* retry(fs.remove(staging)).pipe(Effect.ignore); + } else { + yield* retry(fs.rename(staging, options.target)); + } + yield* Drivers.fsyncDirectory(fs, directory); + }), + // Both modes already remove the staging name on success; only a failed attempt leaves it. + () => retry(fs.remove(staging, { force: true })).pipe(Effect.ignore), + ).pipe(Effect.mapError((cause) => namespaceError("publish", cause))); +}); diff --git a/packages/stack/src/namespace/Registry.ts b/packages/stack/src/namespace/Registry.ts new file mode 100644 index 0000000000..c740d60d25 --- /dev/null +++ b/packages/stack/src/namespace/Registry.ts @@ -0,0 +1,276 @@ +import { Effect, Exit, FileSystem, Path, Schedule, Schema, Scope } from "effect"; +import { CompositionConfig } from "../Orchestrator.ts"; +import { ServiceCreation } from "../services/Catalog.ts"; +import { + isSharingViolation, + namespaceError, + transientRetrySchedule, + type NamespaceError, +} from "./Capabilities.ts"; +import * as Publication from "./Publication.ts"; +import { removeEmptyDirectory } from "./drivers/FileSystem.ts"; +import { acquireLock, isBusy, takeLock } from "./drivers/Sqlite.ts"; + +const SafeId = Schema.String.pipe( + Schema.refine((value): value is string => /^[a-zA-Z0-9_-]+$/u.test(value), { + identifier: "SafeStateId", + message: "Expected a safe state id", + }), +); + +const SavedInstance = Schema.Struct({ + id: SafeId, + creation: Schema.toCodecJson(ServiceCreation), +}); +interface SavedInstance extends Schema.Schema.Type {} + +/** API and JWT signing keys that override the keys a stack derives by default. */ +export const StackKeysInput = Schema.Struct({ + publishableKey: Schema.optionalKey(Schema.String), + secretKey: Schema.optionalKey(Schema.String), + anonKey: Schema.optionalKey(Schema.String), + anonKeyIsOverride: Schema.optionalKey(Schema.Boolean), + serviceRoleKey: Schema.optionalKey(Schema.String), + serviceRoleKeyIsOverride: Schema.optionalKey(Schema.Boolean), + gotrueJwtKeys: Schema.optionalKey(Schema.String), + publicSigningKeys: Schema.optionalKey(Schema.String), + remoteJwks: Schema.optionalKey(Schema.String), +}); +export interface StackKeysInput extends Schema.Schema.Type {} + +export const StackCredentials = Schema.Struct({ + jwtSecret: Schema.String, + postgresRootKey: Schema.String, + databasePassword: Schema.String, + publishableKey: Schema.String, + secretKey: Schema.String, + anonKey: Schema.String, + serviceRoleKey: Schema.String, + jwks: Schema.String, + gotrueJwtKeys: Schema.String, + remoteJwks: Schema.String, + anonKeyIsOverride: Schema.Boolean, + serviceRoleKeyIsOverride: Schema.Boolean, +}); +export interface StackCredentials extends Schema.Schema.Type {} + +export const PortClaim = Schema.Struct({ + key: Schema.String, + host: Schema.String, + port: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 65535 })), +}); +export interface PortClaim extends Schema.Schema.Type {} + +/** A session stack is destroyed when its creator exits; a detached stack outlives it. */ +export const StackLifetime = Schema.Literals(["session", "detached"]); +export type StackLifetime = Schema.Schema.Type; + +export const SavedStack = Schema.Struct({ + id: SafeId, + lifetime: StackLifetime, + identity: Schema.Struct({ + projectRoot: Schema.String, + branchContext: Schema.String, + stackName: Schema.String, + }), + runtime: Schema.Literals(["native", "docker", "podman"]), + instances: Schema.Array(SavedInstance).check( + Schema.makeFilter((instances) => + new Set(instances.map(({ id }) => id)).size === instances.length + ? undefined + : "Expected unique instance ids", + ), + ), + composition: CompositionConfig, + credentials: Schema.optionalKey(StackCredentials), + ports: Schema.Array(PortClaim), +}); +export interface SavedStack extends Schema.Schema.Type {} + +const ClaimsDocument = Schema.Struct({ ports: Schema.Array(PortClaim) }); + +export interface StackClaims { + readonly id: string; + readonly ports: ReadonlyArray; +} + +export interface Interface { + readonly read: (id: string) => Effect.Effect; + /** Skips each stack entry that stays unreadable after transient retries, reporting it to `onInvalidState`. */ + readonly list: Effect.Effect, NamespaceError>; + /** Decodes only each stack's port claims and silently skips entries that cannot provide them. */ + readonly claims: Effect.Effect, NamespaceError>; + readonly save: (state: SavedStack) => Effect.Effect; + readonly remove: (id: string) => Effect.Effect; + /** Not reentrant; wrap metadata updates here, while Ports operations acquire this lock themselves. */ + readonly withLock: ( + effect: Effect.Effect, + ) => Effect.Effect; +} + +const checkId = (id: string): Effect.Effect => + Schema.is(SafeId)(id) + ? Effect.void + : Effect.fail(namespaceError("identity", `Invalid state id: ${id}`)); + +const decodeState = ( + text: string, + id: string, + target: string, +): Effect.Effect => + Schema.decodeEffect(Schema.fromJsonString(SavedStack))(text).pipe( + Effect.mapError((cause) => + namespaceError( + "decode", + `Unable to decode state ${target} for stack ${id}: ${cause.message}`, + ), + ), + ); + +export interface Options { + readonly root: string; + readonly platform?: NodeJS.Platform; + readonly onInvalidState?: (id: string, error: NamespaceError) => Effect.Effect; +} + +export const make = ( + options: Options, +): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = options.root; + const lock = path.join(root, ".registry-lock.sqlite"); + + const stackRoot = (id: string) => path.join(root, id); + const statePath = (id: string) => path.join(stackRoot(id), "state.json"); + const sharingViolation = isSharingViolation(options.platform ?? process.platform); + const retryTransientRead = (effect: Effect.Effect) => + effect.pipe( + Effect.retry({ schedule: transientRetrySchedule, while: sharingViolation }), + Effect.mapError((cause) => namespaceError("read", cause)), + ); + const read = Effect.fn("Namespace.Registry.read")(function* (id: string) { + yield* checkId(id); + const target = statePath(id); + const exists = yield* fs.exists(target).pipe(retryTransientRead); + if (!exists) return undefined; + const text = yield* fs.readFileString(target).pipe(retryTransientRead); + const state = yield* decodeState(text, id, target); + if (state.id !== id) + return yield* namespaceError("identity", "State document identity does not match its path"); + return state; + }); + const stackIds = fs.readDirectory(root).pipe( + Effect.map((entries) => entries.filter(Schema.is(SafeId))), + Effect.mapError((cause) => namespaceError("list", cause)), + ); + const readEntries = ( + readEntry: (id: string) => Effect.Effect, + onSkipped: (id: string, error: NamespaceError) => Effect.Effect, + ) => + Effect.gen(function* () { + const entries: Array = []; + for (const id of yield* stackIds) { + const value = yield* readEntry(id).pipe( + Effect.catch((error) => onSkipped(id, error).pipe(Effect.as(undefined))), + ); + if (value !== undefined) entries.push(value); + } + return entries; + }); + const list = Effect.fn("Namespace.Registry.list")(() => + readEntries(read, (id, error) => options.onInvalidState?.(id, error) ?? Effect.void), + ); + const decodeClaims = Schema.decodeEffect(Schema.fromJsonString(ClaimsDocument)); + const readClaims = (id: string) => + fs.readFileString(statePath(id)).pipe( + retryTransientRead, + Effect.flatMap((text) => + decodeClaims(text).pipe(Effect.mapError((cause) => namespaceError("decode", cause))), + ), + Effect.map(({ ports }): StackClaims => ({ id, ports })), + ); + const claims = Effect.fn("Namespace.Registry.claims")(() => + readEntries(readClaims, () => Effect.void), + ); + const save = Effect.fn("Namespace.Registry.save")(function* (state: SavedStack) { + yield* checkId(state.id); + yield* fs + .makeDirectory(stackRoot(state.id), { recursive: true, mode: 0o700 }) + .pipe(Effect.mapError((cause) => namespaceError("write", cause))); + const serialized = yield* Schema.encodeEffect(Schema.fromJsonString(SavedStack))(state).pipe( + Effect.mapError((cause) => namespaceError("encode", cause)), + ); + yield* Publication.publish(fs, path, { + target: statePath(state.id), + content: serialized, + mode: "replace", + platform: options.platform, + }); + }); + const remove = Effect.fn("Namespace.Registry.remove")(function* (id: string) { + yield* checkId(id); + for (const file of [ + statePath(id), + path.join(stackRoot(id), "owner.json"), + path.join(stackRoot(id), "owner.log"), + ]) + yield* fs + .remove(file, { force: true }) + .pipe(Effect.mapError((cause) => namespaceError("remove", cause))); + yield* removeEmptyDirectory(path.join(stackRoot(id), "data")); + yield* removeEmptyDirectory(stackRoot(id)); + }); + /** + * One attempt: forks `attemptScope` under `guardScope` before opening anything, so there is + * always a scope to close if the open or the `BEGIN IMMEDIATE` below fails or defects. Runs + * uninterruptibly as a single unit; the retry waiting on busy, and `effect` once this + * succeeds, both stay interruptible around it. + */ + const acquireAttempt = ( + guardScope: Scope.Scope, + ): Effect.Effect => + Effect.uninterruptible( + Effect.gen(function* () { + const attemptScope = yield* Scope.fork(guardScope, "sequential"); + yield* acquireLock(lock, "create").pipe( + Scope.provide(attemptScope), + Effect.flatMap(takeLock), + Effect.onExit((exit) => + Exit.isFailure(exit) ? Scope.close(attemptScope, exit) : Effect.void, + ), + ); + return attemptScope; + }), + ); + /** + * Holds the lock in its own standalone scope, so `effect`'s ambient scope (for example a port + * listener it binds) stays whatever the caller already had, not one this lock closes early. + */ + const withLock = Effect.fn("Namespace.Registry.withLock")( + (effect: Effect.Effect) => + Effect.acquireUseRelease( + Scope.make(), + (guardScope) => + acquireAttempt(guardScope).pipe( + Effect.retry({ + schedule: Schedule.spaced("50 millis").pipe( + Schedule.upTo({ duration: "5 seconds" }), + ), + while: isBusy, + }), + Effect.catchIf(isBusy, () => + namespaceError( + "lock", + "Stack registry is locked by another operation; retry shortly", + ), + ), + Effect.andThen(effect), + ), + (guardScope, exit) => Scope.close(guardScope, exit), + ), + ); + + return { read, list: list(), claims: claims(), save, remove, withLock }; + }); diff --git a/packages/stack/src/namespace/drivers/FileSystem.ts b/packages/stack/src/namespace/drivers/FileSystem.ts new file mode 100644 index 0000000000..6e275cb918 --- /dev/null +++ b/packages/stack/src/namespace/drivers/FileSystem.ts @@ -0,0 +1,72 @@ +import { Effect, FileSystem, Path, Predicate } from "effect"; +import type { PlatformError } from "effect/PlatformError"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no non-recursive directory removal operation. +import { rmdir } from "node:fs/promises"; +import { errorCode, namespaceError, type NamespaceError } from "../Capabilities.ts"; + +/** Removes a directory only if it is already empty; any other failure propagates. */ +export const removeEmptyDirectory = (directory: string): Effect.Effect => + Effect.tryPromise({ + try: () => rmdir(directory), + catch: (cause) => namespaceError("cleanup", cause), + }).pipe( + Effect.catchIf( + (error) => + Predicate.hasProperty(error.cause, "code") && + ["ENOENT", "ENOTEMPTY", "EEXIST"].includes(String(error.cause.code)), + () => Effect.void, + ), + ); + +/** Flushes a file's contents to durable storage. */ +export const fsyncFile = ( + fs: FileSystem.FileSystem, + target: string, +): Effect.Effect => + Effect.scoped(fs.open(target, { flag: "r+" }).pipe(Effect.flatMap((file) => file.sync))); + +/** Node reports opening a directory with `fs.open` this way only on Windows. */ +const directoriesUnopenable = (error: PlatformError) => + process.platform === "win32" && errorCode(error) === "EISDIR"; + +/** Flushes a directory's own metadata; fails except for Windows's documented EISDIR limitation. */ +export const fsyncDirectory = ( + fs: FileSystem.FileSystem, + directory: string, +): Effect.Effect => + Effect.scoped(fs.open(directory, { flag: "r" }).pipe(Effect.flatMap((file) => file.sync))).pipe( + Effect.catchIf(directoriesUnopenable, () => Effect.void), + ); + +let probeCounter = 0; + +/** + * Fails before any mutation when `directory`'s filesystem cannot hard-link within itself. + * Unsupported, by maintainer decision, rather than falling back to a non-atomic publish. The probe + * name is unique per call, so concurrent acquisitions under the same root never collide on it. + */ +export const assertHardLinkSupport = Effect.fn("Namespace.assertHardLinkSupport")(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + directory: string, +) { + const probe = path.join(directory, `.namespace-hardlink-probe-${process.pid}-${++probeCounter}`); + const probeLink = `${probe}.link`; + const cleanup = Effect.all( + [ + fs.remove(probe, { force: true }).pipe(Effect.ignore), + fs.remove(probeLink, { force: true }).pipe(Effect.ignore), + ], + { discard: true }, + ); + const supported = yield* fs.writeFileString(probe, "").pipe( + Effect.mapError((cause) => namespaceError("hardlink-support", cause)), + Effect.andThen(fs.link(probe, probeLink).pipe(Effect.isSuccess)), + Effect.ensuring(cleanup), + ); + if (!supported) + return yield* namespaceError( + "hardlink-support", + `${directory} is on a filesystem without hard-link support, which the stack namespace requires for atomic publication`, + ); +}); diff --git a/packages/stack/src/namespace/drivers/Sqlite.ts b/packages/stack/src/namespace/drivers/Sqlite.ts new file mode 100644 index 0000000000..2280a3825b --- /dev/null +++ b/packages/stack/src/namespace/drivers/Sqlite.ts @@ -0,0 +1,73 @@ +// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no OS-owned cross-process lock primitive. +import { DatabaseSync } from "node:sqlite"; +import { pathToFileURL } from "node:url"; +import { Effect, Predicate, Scope } from "effect"; +import { namespaceError, type NamespaceError } from "../Capabilities.ts"; + +const openConnection = (path: string, mode: "create" | "existing") => + Effect.try({ + try: () => + mode === "create" + ? new DatabaseSync(path) + : new DatabaseSync(new URL(`${pathToFileURL(path).href}?mode=rw`)), + catch: (cause) => namespaceError("lock-open", cause), + }); +const closeConnection = (connection: DatabaseSync) => + Effect.try({ + try: () => connection.close(), + catch: (cause) => namespaceError("lock-close", cause), + }); + +/** Matches a `NamespaceError`'s cause against one of `node:sqlite`'s numeric `errcode`s. */ +export const errcode = (cause: unknown, code: number) => + Predicate.hasProperty(cause, "errcode") && cause.errcode === code; +/** SQLite reports lock contention under `busy_timeout = 0` with this errcode. */ +export const isBusy = (error: NamespaceError) => errcode(error.cause, 5); +const busy = (operation: string) => + namespaceError(operation, Object.assign(new Error("locked by this process"), { errcode: 5 })); + +/** + * Canonical paths this process currently has a connection open on. POSIX advisory locks (and the + * SQLite locks built on them) belong to a process, not a descriptor: a second connection this + * process opened on the same path would not see the first one's lock as contention, so it would + * wrongly appear to succeed instead of reporting busy. Checking this set first, before opening a + * second descriptor, rules that out: a path already open in this process is always busy. + */ +const openPaths = new Set(); + +/** + * Opens `path`'s connection for the enclosing scope; busy if this process already has it open. + * Reserving the path and opening the connection happen as one `acquireRelease` acquisition, so an + * interruption can never leave the path reserved without a connection whose release clears it. + */ +export const acquireLock = ( + path: string, + mode: "create" | "existing", +): Effect.Effect => + Effect.gen(function* () { + if (openPaths.has(path)) return yield* busy("lock"); + return yield* Effect.acquireRelease( + Effect.sync(() => openPaths.add(path)).pipe( + Effect.andThen(openConnection(path, mode)), + Effect.tapError(() => Effect.sync(() => openPaths.delete(path))), + ), + (connection) => + closeConnection(connection).pipe( + Effect.catch((error) => Effect.logWarning(`Unable to close ${path}`, error)), + Effect.ensuring(Effect.sync(() => openPaths.delete(path))), + ), + ); + }); + +/** Takes the connection's write lock once; fails busy under any contention instead of waiting. */ +export const takeLock = (connection: DatabaseSync): Effect.Effect => + Effect.gen(function* () { + yield* Effect.try({ + try: () => connection.exec("PRAGMA busy_timeout = 0"), + catch: (cause) => namespaceError("lock", cause), + }); + yield* Effect.try({ + try: () => connection.exec("BEGIN IMMEDIATE"), + catch: (cause) => namespaceError("lock", cause), + }); + }); diff --git a/packages/stack/src/services/Initialization.ts b/packages/stack/src/services/Initialization.ts index 7e67c439f5..591a1c7f05 100644 --- a/packages/stack/src/services/Initialization.ts +++ b/packages/stack/src/services/Initialization.ts @@ -2,7 +2,7 @@ import { Effect } from "effect"; import { resolveArtifact } from "../Artifacts.ts"; import type { InitializationCommand, ResolvedCommand } from "../Commands.ts"; import { ServiceError } from "../Service.ts"; -import type { StackCredentials } from "../State.ts"; +import type { StackCredentials } from "../StackNamespace.ts"; import * as Auth from "./Auth.ts"; import * as Realtime from "./Realtime.ts"; import * as Storage from "./Storage.ts"; diff --git a/packages/stack/src/services/Rest.integration.test.ts b/packages/stack/src/services/Rest.integration.test.ts index a8d87db95f..779a79e542 100644 --- a/packages/stack/src/services/Rest.integration.test.ts +++ b/packages/stack/src/services/Rest.integration.test.ts @@ -6,25 +6,25 @@ import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { SignJWT } from "jose"; import { tmpdir } from "node:os"; import * as Network from "../Network.ts"; -import * as State from "../State.ts"; +import * as StackNamespace from "../StackNamespace.ts"; import { makeService } from "../Service.ts"; import { ProxyError } from "../Proxy.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; const makeTestState = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), ); const makeTestNetwork = (options: { readonly stackId: string; readonly runtime: Network.NetworkRuntime; - readonly state: State.Interface; + readonly state: StackNamespace.Interface; }) => Layer.build( Network.layer({ stackId: options.stackId, runtime: options.runtime }).pipe( - Layer.provide(Layer.succeed(State.Service, options.state)), + Layer.provide(Layer.succeed(StackNamespace.Service, options.state)), ), ).pipe(Effect.map((context) => Context.get(context, Network.Service))); diff --git a/packages/stack/src/services/ServiceConfig.ts b/packages/stack/src/services/ServiceConfig.ts index 72156d14a0..58ee7f9f66 100644 --- a/packages/stack/src/services/ServiceConfig.ts +++ b/packages/stack/src/services/ServiceConfig.ts @@ -7,7 +7,7 @@ import { DEFAULT_LOCAL_SECRET_KEY, DEFAULT_SIGNING_KEY, } from "../Defaults.ts"; -import type { StackCredentials, StackKeysInput } from "../State.ts"; +import type { StackCredentials, StackKeysInput } from "../StackNamespace.ts"; const serviceError = (operation: string, cause: unknown): ServiceError => cause instanceof ServiceError diff --git a/packages/stack/src/services/StorageGateway.integration.test.ts b/packages/stack/src/services/StorageGateway.integration.test.ts index 857e6b9460..ed308311eb 100644 --- a/packages/stack/src/services/StorageGateway.integration.test.ts +++ b/packages/stack/src/services/StorageGateway.integration.test.ts @@ -8,8 +8,8 @@ import { DEFAULT_LOCAL_S3_REGION, DEFAULT_LOCAL_S3_SECRET_ACCESS_KEY, } from "../Defaults.ts"; -import * as State from "../State.ts"; -import type { SavedStack } from "../State.ts"; +import * as StackNamespace from "../StackNamespace.ts"; +import type { SavedStack } from "../StackNamespace.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { ownerFor } from "../../tests/owner-rpc.ts"; @@ -54,7 +54,10 @@ const serveStorage = Effect.fnUntraced(function* (runtime: SavedStack["runtime"] composition: { members: [], dependencies: [] }, ports: [], }; - const state = Context.get(yield* Layer.build(State.layer({ root: stateRoot })), State.Service); + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root: stateRoot })), + StackNamespace.Service, + ); yield* state.save(saved); const owner = yield* ownerFor({ saved, state, root: dataRoot, cacheRoot }); yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); diff --git a/packages/stack/src/shutdown.integration.test.ts b/packages/stack/src/shutdown.integration.test.ts index 905d95750c..97da9b095a 100644 --- a/packages/stack/src/shutdown.integration.test.ts +++ b/packages/stack/src/shutdown.integration.test.ts @@ -6,7 +6,7 @@ import { fileURLToPath } from "node:url"; import * as PromiseStack from "./index.ts"; import { HostEndpoint } from "./HostProcess.ts"; import { StackError } from "./Rpc.ts"; -import * as State from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; import { open as openEffect } from "./effect.ts"; const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); @@ -51,8 +51,8 @@ const withHeldOwner = ( const stateRoot = `${root}/state`; const cacheRoot = `${root}/cache`; const id = "shutdown-held"; - const context = yield* Layer.build(State.layer({ root: stateRoot })); - const state = Context.get(context, State.Service); + const context = yield* Layer.build(StackNamespace.layer({ root: stateRoot })); + const state = Context.get(context, StackNamespace.Service); yield* state.save({ id, runtime: "native", diff --git a/packages/stack/tests/compiled-dispatch-fixture.ts b/packages/stack/tests/compiled-dispatch-fixture.ts index a4ac872efd..ff2f35a3fd 100644 --- a/packages/stack/tests/compiled-dispatch-fixture.ts +++ b/packages/stack/tests/compiled-dispatch-fixture.ts @@ -13,7 +13,7 @@ import { type HostEndpoint, } from "../src/HostProcess.ts"; import { spawnNativeProcess } from "../src/runtime/NativeProcess.ts"; -import * as State from "../src/State.ts"; +import * as StackNamespace from "../src/StackNamespace.ts"; const argv = process.argv.slice(2); class FixtureError extends Data.TaggedError("FixtureError")<{ readonly message: string }> {} @@ -27,7 +27,7 @@ if (!(await runHostProcessIfDispatched(argv)) && !(await runNativeProcessIfDispa throw new Error("Owner fixture arguments missing"); const program = Effect.scoped( Effect.gen(function* () { - const state = yield* State.Service; + const state = yield* StackNamespace.Service; const access = yield* mode === "owner" ? launchHost(state, { stateRoot, cacheRoot: cacheRoot ?? stateRoot, stackId }) : connectHost(state, stackId); @@ -47,7 +47,7 @@ if (!(await runHostProcessIfDispatched(argv)) && !(await runNativeProcessIfDispa Layer.mergeAll( NodeServices.layer, NodeHttpClient.layerNodeHttp, - State.layer({ root: stateRoot }).pipe(Layer.provide(NodeServices.layer)), + StackNamespace.layer({ root: stateRoot }).pipe(Layer.provide(NodeServices.layer)), ), ), ); diff --git a/packages/stack/tests/host-process-fixture.ts b/packages/stack/tests/host-process-fixture.ts index f0210334c5..ba571ad63e 100644 --- a/packages/stack/tests/host-process-fixture.ts +++ b/packages/stack/tests/host-process-fixture.ts @@ -16,13 +16,13 @@ import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; import { closeSync, createReadStream, writeSync } from "node:fs"; import { currentRelease, launchHost, authorizes, type HostEndpoint } from "../src/HostProcess.ts"; import { bindControl } from "../src/StackHost.ts"; -import * as State from "../src/State.ts"; +import * as StackNamespace from "../src/StackNamespace.ts"; class FixtureError extends Data.TaggedError("FixtureError")<{ readonly message: string }> {} const makeState = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), ); const awaitBarrier = NodeStream.fromReadable({ @@ -56,11 +56,15 @@ const owner = Effect.scoped( const path = yield* Path.Path; const stack = yield* state.read(stackId); if (stack === undefined) return yield* new FixtureError({ message: "stack is not registered" }); - if (!(yield* state.lease(stackId))) { - report({ type: "error", message: "lease held", reason: "lease-held" }); - return; - } - yield* state.retractHolder(stackId); + const lease = yield* state.acquireLease(stackId).pipe( + Effect.catchTag("Namespace.LeaseHeldError", () => + Effect.sync(() => { + report({ type: "error", message: "lease held", reason: "lease-held" }); + }).pipe(Effect.as(undefined)), + ), + ); + if (lease === undefined) return; + yield* lease.retractHolder; if (stack.identity.stackName === "slow-handshake") { const marker = path.join(stateRoot, "slow-handshake.pid"); yield* fs.writeFileString(`${marker}.tmp`, String(process.pid)); @@ -97,7 +101,7 @@ const owner = Effect.scoped( }), ) .pipe(Effect.forkScoped); - yield* state.publishHolder(stackId, { + yield* lease.publishHolder({ role: "owner", secret, port: endpoint.port, @@ -109,7 +113,7 @@ const owner = Effect.scoped( report({ type: "ready", endpoint, secret }); if (mode === "held") yield* awaitBarrier; else yield* Deferred.await(shutdown); - yield* state.retractHolder(stackId); + yield* lease.retractHolder; }), ); diff --git a/packages/stack/tests/state-lock-fixture.ts b/packages/stack/tests/namespace-lock-fixture.ts similarity index 69% rename from packages/stack/tests/state-lock-fixture.ts rename to packages/stack/tests/namespace-lock-fixture.ts index d78b5d0e24..322b15e7be 100644 --- a/packages/stack/tests/state-lock-fixture.ts +++ b/packages/stack/tests/namespace-lock-fixture.ts @@ -1,9 +1,8 @@ import { NodeServices, NodeSocketServer } from "@effect/platform-node"; import { Console, Effect } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import * as State from "../src/State.ts"; -import { isBunVirtualPath } from "../src/internal/dispatch-markers.ts"; import { fileURLToPath } from "node:url"; +import * as StackNamespace from "../src/StackNamespace.ts"; const [mode, root, id] = process.argv.slice(2); if (root === undefined) throw new Error("Registry root missing"); @@ -19,7 +18,7 @@ const program = Effect.scoped( return yield* Effect.never; } yield* Effect.gen(function* () { - const state = yield* State.Service; + const state = yield* StackNamespace.Service; yield* state.withLock( Effect.gen(function* () { if (mode === "write") { @@ -34,24 +33,20 @@ const program = Effect.scoped( } const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; yield* spawner.spawn( - ChildProcess.make( - process.execPath, - [...(isBunVirtualPath(fixturePath) ? [] : [fixturePath]), "child", root], - { - // Keep the child independent on Windows so holder termination tests lock recovery. - detached: process.platform === "win32", - stdin: "ignore", - stdout: "inherit", - stderr: "inherit", - forceKillAfter: "1 second", - }, - ), + ChildProcess.make(process.execPath, [fixturePath, "child", root], { + // Keep the child independent on Windows so holder termination tests lock recovery. + detached: process.platform === "win32", + stdin: "ignore", + stdout: "inherit", + stderr: "inherit", + forceKillAfter: "1 second", + }), ); yield* Console.log("locked"); return yield* Effect.never; }), ); - }).pipe(Effect.provide(State.layer({ root }))); + }).pipe(Effect.provide(StackNamespace.layer({ root }))); }), ).pipe(Effect.provide(NodeServices.layer)); diff --git a/packages/stack/tests/owner-rpc.ts b/packages/stack/tests/owner-rpc.ts index fd8d254423..50652a6db0 100644 --- a/packages/stack/tests/owner-rpc.ts +++ b/packages/stack/tests/owner-rpc.ts @@ -2,8 +2,8 @@ import { Context, Effect, Layer } from "effect"; import { RpcTest } from "effect/unstable/rpc"; import * as Owner from "../src/Owner.ts"; import { OwnerRpc } from "../src/Rpc.ts"; -import { Service as StateService } from "../src/State.ts"; -import type { Interface as StateInterface, SavedStack } from "../src/State.ts"; +import { Service as StateService } from "../src/StackNamespace.ts"; +import type { Interface as StateInterface, SavedStack } from "../src/StackNamespace.ts"; /** Builds an in-process owner and an RPC client bound to its handlers. */ export const ownerFor = (options: { diff --git a/packages/stack/tests/publish-loop-fixture.ts b/packages/stack/tests/publish-loop-fixture.ts new file mode 100644 index 0000000000..a6bb57e3bc --- /dev/null +++ b/packages/stack/tests/publish-loop-fixture.ts @@ -0,0 +1,40 @@ +import { NodeServices } from "@effect/platform-node"; +import { Console, Effect, FileSystem, Layer } from "effect"; +import * as StackNamespace from "../src/StackNamespace.ts"; + +const [root, id] = process.argv.slice(2); +if (root === undefined || id === undefined) throw new Error("Registry root or stack id missing"); + +/** + * Publishes one revision of the saved stack, announcing readiness on stdout the instant it is + * about to replace the target document (the real staging-to-publish boundary Publication.publish + * uses), then hangs so a caller can kill this process exactly there on every run. + */ +const barrierFs = Layer.effect( + FileSystem.FileSystem, + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + return { + ...fs, + rename: () => Console.log("about-to-publish").pipe(Effect.andThen(Effect.never)), + }; + }), +); + +const program = Effect.scoped( + Effect.gen(function* () { + const state = yield* StackNamespace.Service; + const saved = yield* state.read(id); + if (saved === undefined) return yield* Effect.die(`Stack ${id} is not registered`); + yield* state.save({ ...saved, identity: { ...saved.identity, stackName: "revision-1" } }); + }), +).pipe( + Effect.provide( + StackNamespace.layer({ root }).pipe( + Layer.provide(barrierFs), + Layer.provide(NodeServices.layer), + ), + ), +); + +await Effect.runPromise(program); From 3254e7d2bbfeb26b95e68af8b5766b64e59697b5 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sat, 3 Oct 2026 05:55:16 +0200 Subject: [PATCH 007/142] fix(stack): skip directory fsync on Windows and track Schema.TaggedError tags --- .../telemetry/__fixtures__/error-tags.txt | 6 ++ .../error-tag-stability.unit.test.ts | 12 +-- .../namespace/Namespace.integration.test.ts | 93 ++++++++++--------- .../stack/src/namespace/drivers/FileSystem.ts | 14 +-- 4 files changed, 65 insertions(+), 60 deletions(-) diff --git a/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt b/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt index fc5ef75610..8dde3a4673 100644 --- a/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt +++ b/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt @@ -253,10 +253,16 @@ FeedbackInvalidTokenError FeedbackMessageTooLongError FeedbackNotFoundError FileWatcherError +FunctionAssetIsDirectoryError +FunctionAssetOutsideRootError +FunctionBundleFailedError FunctionDeployCancelledError +FunctionDeployJobsRequiresApiError FunctionDownloadNotFoundError +FunctionImportMapParseError FunctionImportNotDirectoryError FunctionNotFoundError +FunctionStaticPatternNoMatchError FunctionsApiStatusError FunctionsApiTransportError FunctionsEnvNotSupportedError diff --git a/apps/cli/src/shared/telemetry/error-tag-stability.unit.test.ts b/apps/cli/src/shared/telemetry/error-tag-stability.unit.test.ts index 5b4a4e3620..a3e24013e6 100644 --- a/apps/cli/src/shared/telemetry/error-tag-stability.unit.test.ts +++ b/apps/cli/src/shared/telemetry/error-tag-stability.unit.test.ts @@ -5,11 +5,11 @@ import { fileURLToPath, pathToFileURL } from "node:url"; /** * Guards the telemetry identity of every CLI error: the string passed to - * `Data.TaggedError("...")` becomes `error_fingerprint` on the `cli_command_executed` event, so + * `Data.TaggedError("...")` or `Schema.TaggedError` becomes `error_fingerprint` on the `cli_command_executed` event, so * changing it (not renaming the class) silently splits one error's history into two * fingerprints, undetected until dashboards look wrong. * - * Enumerates every production `Data.TaggedError` tag under `apps/cli/src` and + * Enumerates every production tagged-error tag under `apps/cli/src` and * `packages/config/src`, resolving tags built via template interpolation * (`mintConfigTargetErrors`) at runtime by constructing each class — see * {@link collectComputedTagDeclarations}. Compares the set against the committed @@ -26,10 +26,10 @@ const externalConfigSrcDir = fileURLToPath( ); const fixturePath = fileURLToPath(new URL("./__fixtures__/error-tags.txt", import.meta.url)); -// Matches both the inline declaration form and the formatter-wrapped -// multi-line form, where the string literal lands on its own line before the -// closing paren. -const TAGGED_ERROR_PATTERN = /class\s+(\w+)\s+extends\s+Data\.TaggedError\(\s*["']([^"']+)["']/g; +// Matches `Data.TaggedError("…")` and `Schema.TaggedError()("…", …)`, inline or +// formatter-wrapped with the string literal on its own line. +const TAGGED_ERROR_PATTERN = + /class\s+(\w+)\s+extends\s+(?:Data\.TaggedError|Schema\.TaggedError<\w+>\(\))\(\s*["']([^"']+)["']/g; // Every test tier this repo's vitest config defines (`apps/cli/vitest.config.ts`). // Shared by name so a fifth tier can't quietly slip past `isProductionSourceFile` diff --git a/packages/stack/src/namespace/Namespace.integration.test.ts b/packages/stack/src/namespace/Namespace.integration.test.ts index dcd2ddf6c4..3133726b4d 100644 --- a/packages/stack/src/namespace/Namespace.integration.test.ts +++ b/packages/stack/src/namespace/Namespace.integration.test.ts @@ -1001,51 +1001,54 @@ describe("publication", () => { ), ); - it.effect("fails publication when the directory fsync reports EIO", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const eio = (method: string, file: string) => - PlatformError.systemError({ - _tag: "Unknown", - module: "FileSystem", - method, - pathOrDescriptor: file, - cause: Object.assign(new Error("injected directory fsync failure"), { code: "EIO" }), - }); - // Covers both ways opening the directory for its own fsync can report EIO: the `open` - // call itself failing, and `open` succeeding but the handle's own `sync` failing. The - // second case is backed by a real regular-file handle (scoped like any other), not the - // real directory: opening a directory for `sync` hits Windows's own supported `EISDIR` - // path instead of ever reaching the injected failure. - for (const mode of ["open", "sync"] as const) { - const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-fsync-eio-" }); - const directory = path.join(root, initial.id); - const syncProbe = path.join(root, ".sync-probe"); - yield* fs.writeFileString(syncProbe, ""); - const failingFs = Layer.succeed(FileSystem.FileSystem, { - ...fs, - open: (file: string, options?: Parameters[1]) => { - if (file !== directory || options?.flag !== "r") return fs.open(file, options); - if (mode === "open") return Effect.fail(eio("open", file)); - return fs - .open(syncProbe, options) - .pipe( - Effect.map((handle) => ({ ...handle, sync: Effect.fail(eio("fsync", file)) })), - ); - }, - }); - const store = yield* Layer.build( - StackNamespace.layer({ root }).pipe(Layer.provide(failingFs)), - ).pipe(Effect.map((context) => Context.get(context, StackNamespace.Service))); - const failure = yield* store.save(initial).pipe(Effect.flip); - expect(failure, mode).toBeInstanceOf(StackNamespace.NamespaceError); - expect(failure.operation, mode).toBe("publish"); - expect(errorCode(failure.cause), mode).toBe("EIO"); - } - }), - ), + // Windows has no directory fsync to fail. + it.effect.skipIf(process.platform === "win32")( + "fails publication when the directory fsync reports EIO", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const eio = (method: string, file: string) => + PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method, + pathOrDescriptor: file, + cause: Object.assign(new Error("injected directory fsync failure"), { code: "EIO" }), + }); + // Covers both ways opening the directory for its own fsync can report EIO: the `open` + // call itself failing, and `open` succeeding but the handle's own `sync` failing. The + // second case is backed by a real regular-file handle (scoped like any other), not the + // real directory: opening a directory for `sync` hits Windows's own supported `EISDIR` + // path instead of ever reaching the injected failure. + for (const mode of ["open", "sync"] as const) { + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-fsync-eio-" }); + const directory = path.join(root, initial.id); + const syncProbe = path.join(root, ".sync-probe"); + yield* fs.writeFileString(syncProbe, ""); + const failingFs = Layer.succeed(FileSystem.FileSystem, { + ...fs, + open: (file: string, options?: Parameters[1]) => { + if (file !== directory || options?.flag !== "r") return fs.open(file, options); + if (mode === "open") return Effect.fail(eio("open", file)); + return fs + .open(syncProbe, options) + .pipe( + Effect.map((handle) => ({ ...handle, sync: Effect.fail(eio("fsync", file)) })), + ); + }, + }); + const store = yield* Layer.build( + StackNamespace.layer({ root }).pipe(Layer.provide(failingFs)), + ).pipe(Effect.map((context) => Context.get(context, StackNamespace.Service))); + const failure = yield* store.save(initial).pipe(Effect.flip); + expect(failure, mode).toBeInstanceOf(StackNamespace.NamespaceError); + expect(failure.operation, mode).toBe("publish"); + expect(errorCode(failure.cause), mode).toBe("EIO"); + } + }), + ), ); it.effect("leaves no staging file after a partial write fails with ENOSPC", () => diff --git a/packages/stack/src/namespace/drivers/FileSystem.ts b/packages/stack/src/namespace/drivers/FileSystem.ts index 6e275cb918..de9f47bc83 100644 --- a/packages/stack/src/namespace/drivers/FileSystem.ts +++ b/packages/stack/src/namespace/drivers/FileSystem.ts @@ -2,7 +2,7 @@ import { Effect, FileSystem, Path, Predicate } from "effect"; import type { PlatformError } from "effect/PlatformError"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no non-recursive directory removal operation. import { rmdir } from "node:fs/promises"; -import { errorCode, namespaceError, type NamespaceError } from "../Capabilities.ts"; +import { namespaceError, type NamespaceError } from "../Capabilities.ts"; /** Removes a directory only if it is already empty; any other failure propagates. */ export const removeEmptyDirectory = (directory: string): Effect.Effect => @@ -25,18 +25,14 @@ export const fsyncFile = ( ): Effect.Effect => Effect.scoped(fs.open(target, { flag: "r+" }).pipe(Effect.flatMap((file) => file.sync))); -/** Node reports opening a directory with `fs.open` this way only on Windows. */ -const directoriesUnopenable = (error: PlatformError) => - process.platform === "win32" && errorCode(error) === "EISDIR"; - -/** Flushes a directory's own metadata; fails except for Windows's documented EISDIR limitation. */ +/** Flushes a directory's own metadata; a no-op on Windows, which can't open or fsync directories. */ export const fsyncDirectory = ( fs: FileSystem.FileSystem, directory: string, ): Effect.Effect => - Effect.scoped(fs.open(directory, { flag: "r" }).pipe(Effect.flatMap((file) => file.sync))).pipe( - Effect.catchIf(directoriesUnopenable, () => Effect.void), - ); + process.platform === "win32" + ? Effect.void + : Effect.scoped(fs.open(directory, { flag: "r" }).pipe(Effect.flatMap((file) => file.sync))); let probeCounter = 0; From c28bce84dddc2440e282e26dcb7574afdedf16d8 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sat, 3 Oct 2026 06:30:45 +0200 Subject: [PATCH 008/142] fix(stack): replace the lease holder record atomically and drop the hard-link requirement --- packages/stack/src/StackNamespace.ts | 2 - packages/stack/src/namespace/Lease.ts | 20 +- .../namespace/Namespace.integration.test.ts | 229 +++++++++--------- packages/stack/src/namespace/Publication.ts | 24 +- packages/stack/src/namespace/Registry.ts | 1 - .../stack/src/namespace/drivers/FileSystem.ts | 35 +-- 6 files changed, 137 insertions(+), 174 deletions(-) diff --git a/packages/stack/src/StackNamespace.ts b/packages/stack/src/StackNamespace.ts index ec892dc158..bacf8ada1a 100644 --- a/packages/stack/src/StackNamespace.ts +++ b/packages/stack/src/StackNamespace.ts @@ -1,6 +1,5 @@ import { Context, Effect, FileSystem, Layer, Path } from "effect"; import { namespaceError, type NamespaceError } from "./namespace/Capabilities.ts"; -import * as FsDriver from "./namespace/drivers/FileSystem.ts"; import * as Lease from "./namespace/Lease.ts"; import * as Registry from "./namespace/Registry.ts"; import { restrictDirectoryToOwner } from "./runtime/postgres-user.ts"; @@ -42,7 +41,6 @@ const make = Effect.fn("Namespace.acquire")(function* ( yield* restrictDirectoryToOwner(fs, root).pipe( Effect.mapError((cause) => namespaceError("root", cause)), ); - yield* FsDriver.assertHardLinkSupport(fs, path, root); const registry = yield* Registry.make({ root, platform: options.platform, diff --git a/packages/stack/src/namespace/Lease.ts b/packages/stack/src/namespace/Lease.ts index 7afab4a968..9ec17a6d6e 100644 --- a/packages/stack/src/namespace/Lease.ts +++ b/packages/stack/src/namespace/Lease.ts @@ -238,19 +238,15 @@ export const make = ( const serialized = yield* Schema.encodeEffect(Schema.fromJsonString(LeaseHolder))( record, ).pipe(Effect.mapError((cause) => namespaceError("encode", cause))); + // The lease already guarantees a single writer, so an atomic rename over any existing + // record (a live one, or a crashed holder's stale one) is enough: a reader never sees a + // gap where the file briefly doesn't exist, unlike a prior remove followed by a publish. yield* guarded( - // A crashed holder's stale record, if any, must not block this fresh one from publishing. - fs.remove(ownerPath(id), { force: true }).pipe( - Effect.mapError((cause) => namespaceError("remove", cause)), - Effect.andThen( - Publication.publish(fs, path, { - target: ownerPath(id), - content: serialized, - mode: "initial", - platform: options.platform, - }), - ), - ), + Publication.publish(fs, path, { + target: ownerPath(id), + content: serialized, + platform: options.platform, + }), ); }), retractHolder: guarded( diff --git a/packages/stack/src/namespace/Namespace.integration.test.ts b/packages/stack/src/namespace/Namespace.integration.test.ts index 3133726b4d..6a82787c25 100644 --- a/packages/stack/src/namespace/Namespace.integration.test.ts +++ b/packages/stack/src/namespace/Namespace.integration.test.ts @@ -13,7 +13,6 @@ import { Path, PlatformError, Ref, - Result, Schema, Scheduler, Scope, @@ -639,21 +638,21 @@ describe("stack owner lease", () => { // finalizer below can only proceed once A's publish has fully released the gate. const paused = yield* Deferred.make(); const entered = yield* Deferred.make(); - const removeCalls = yield* Ref.make(0); + const renameCalls = yield* Ref.make(0); const pausingFs = Layer.effect( FileSystem.FileSystem, Effect.sync(() => ({ ...fs, - remove: (file: string, options?: Parameters[1]) => + rename: (oldPath: string, newPath: string) => Effect.gen(function* () { const isFirst = - file === ownerJson && - (yield* Ref.getAndUpdate(removeCalls, (n) => n + 1)) === 0; + newPath === ownerJson && + (yield* Ref.getAndUpdate(renameCalls, (n) => n + 1)) === 0; if (isFirst) { yield* Deferred.succeed(entered, undefined); yield* Deferred.await(paused); } - return yield* fs.remove(file, options); + return yield* fs.rename(oldPath, newPath); }), })), ); @@ -866,64 +865,122 @@ describe("stack owner lease", () => { ), ); - it.live( - "publishes successfully and cleans up staging despite a transient staging-removal failure", - () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const id = "flaky-cleanup"; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-publish-cleanup-" }); - const stackDirectory = path.join(root, id); - // A single ignored attempt (the pre-fix behavior) only ever makes one call per site; - // three consecutive failures need a real retry loop to still converge. - const remaining = yield* Ref.make(3); - const flakyFs = Layer.succeed(FileSystem.FileSystem, { - ...fs, - remove: (file: string, options?: Parameters[1]) => - Effect.gen(function* () { - if ( - file.startsWith(path.join(stackDirectory, ".owner.json.")) && - (yield* Ref.get(remaining)) > 0 - ) { - yield* Ref.update(remaining, (count) => count - 1); - return yield* PlatformError.systemError({ - _tag: "Unknown", - module: "FileSystem", - method: "remove", - pathOrDescriptor: file, - cause: Object.assign(new Error("injected transient removal failure"), { - code: "EBUSY", - }), - }); - } - return yield* fs.remove(file, options); - }), - }); - const state = Context.get( - yield* Layer.build( - StackNamespace.layer({ root, platform: "win32" }).pipe(Layer.provide(flakyFs)), - ), - StackNamespace.Service, - ); - const scope = yield* Scope.make(); - const lease = yield* state.acquireLease(id).pipe(Scope.provide(scope)); - const record: StackNamespace.LeaseHolder = { - role: "sweeper", - pid: process.pid, - startedAt: "2026-01-01T00:00:00.000Z", - }; - yield* lease.publishHolder(record); - - expect(yield* Ref.get(remaining), "all injected failures were consumed").toBe(0); - expect(yield* state.readHolder(id)).toEqual(record); - expect( - (yield* fs.readDirectory(stackDirectory)).filter((entry) => entry.endsWith(".tmp")), - ).toEqual([]); - yield* Scope.close(scope, Exit.void); - }), - ), + it.live("publishes successfully despite a transient rename failure", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const id = "flaky-cleanup"; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-publish-cleanup-" }); + const stackDirectory = path.join(root, id); + const ownerJson = path.join(stackDirectory, "owner.json"); + // A single ignored attempt (the pre-fix behavior) only ever makes one call per site; + // three consecutive failures need a real retry loop to still converge. + const remaining = yield* Ref.make(3); + const flakyFs = Layer.succeed(FileSystem.FileSystem, { + ...fs, + rename: (oldPath: string, newPath: string) => + Effect.gen(function* () { + if (newPath === ownerJson && (yield* Ref.get(remaining)) > 0) { + yield* Ref.update(remaining, (count) => count - 1); + return yield* PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method: "rename", + pathOrDescriptor: newPath, + cause: Object.assign(new Error("injected transient rename failure"), { + code: "EBUSY", + }), + }); + } + return yield* fs.rename(oldPath, newPath); + }), + }); + const state = Context.get( + yield* Layer.build( + StackNamespace.layer({ root, platform: "win32" }).pipe(Layer.provide(flakyFs)), + ), + StackNamespace.Service, + ); + const scope = yield* Scope.make(); + const lease = yield* state.acquireLease(id).pipe(Scope.provide(scope)); + const record: StackNamespace.LeaseHolder = { + role: "sweeper", + pid: process.pid, + startedAt: "2026-01-01T00:00:00.000Z", + }; + yield* lease.publishHolder(record); + + expect(yield* Ref.get(remaining), "all injected failures were consumed").toBe(0); + expect(yield* state.readHolder(id)).toEqual(record); + expect( + (yield* fs.readDirectory(stackDirectory)).filter((entry) => entry.endsWith(".tmp")), + ).toEqual([]); + yield* Scope.close(scope, Exit.void); + }), + ), + ); + + it.live("never reports a holder record missing while replacing it with a fresh one", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const id = "atomic-replace"; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lease-atomic-" }); + const ownerJson = path.join(root, id, "owner.json"); + const gate = yield* Deferred.make(); + const entered = yield* Deferred.make(); + // Only the second publish (the replacement) is gated; the first establishes R1 normally. + let publishCount = 0; + const gatedFs = Layer.succeed(FileSystem.FileSystem, { + ...fs, + rename: (oldPath: string, newPath: string) => { + if (newPath !== ownerJson) return fs.rename(oldPath, newPath); + publishCount++; + return publishCount === 2 + ? Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(gate)), + Effect.andThen(fs.rename(oldPath, newPath)), + ) + : fs.rename(oldPath, newPath); + }, + }); + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root }).pipe(Layer.provide(gatedFs))), + StackNamespace.Service, + ); + const scope = yield* Scope.make(); + const lease = yield* state.acquireLease(id).pipe(Scope.provide(scope)); + const r1: StackNamespace.LeaseHolder = { + role: "sweeper", + pid: process.pid, + startedAt: "2026-01-01T00:00:00.000Z", + }; + yield* lease.publishHolder(r1); + expect(yield* state.readHolder(id)).toEqual(r1); + + const r2: StackNamespace.LeaseHolder = { + role: "sweeper", + pid: process.pid + 1, + startedAt: "2026-01-01T00:00:01.000Z", + }; + const publishing = yield* lease.publishHolder(r2).pipe(Effect.forkScoped); + // Runs before the publisher's interruption (reverse order), so a failed assertion can't + // leave the uninterruptible publish blocked on the gate during teardown. + yield* Effect.addFinalizer(() => Deferred.succeed(gate, undefined)); + yield* Deferred.await(entered); + + // At the publication gate (the rename not yet landed), a reader must still see the prior + // record, never a gap: a remove-then-publish sequence would briefly have no file at all. + expect(yield* state.readHolder(id)).toEqual(r1); + + yield* Deferred.succeed(gate, undefined); + yield* Fiber.join(publishing); + expect(yield* state.readHolder(id)).toEqual(r2); + yield* Scope.close(scope, Exit.void); + }), + ), ); }); @@ -957,50 +1014,6 @@ describe("discovery", () => { }); describe("publication", () => { - it.effect("rejects a state root without hard-link support before any mutation", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-nolink-" }); - const noLink = Layer.succeed(FileSystem.FileSystem, { - ...fs, - link: (_fromPath: string, toPath: string) => - Effect.fail( - PlatformError.systemError({ - _tag: "Unknown", - module: "FileSystem", - method: "link", - pathOrDescriptor: toPath, - description: "Hard links are not supported on this filesystem", - cause: Object.assign(new Error("link not supported"), { code: "ENOTSUP" }), - }), - ), - }); - const failure = yield* StackNamespace.Service.pipe( - Effect.provide(StackNamespace.layer({ root }).pipe(Layer.provide(noLink))), - Effect.flip, - ); - expect(failure).toBeInstanceOf(StackNamespace.NamespaceError); - expect(failure.operation).toBe("hardlink-support"); - expect(yield* fs.readDirectory(root)).toEqual([]); - }), - ), - ); - - it.live("accepts a root that supports hard links under concurrent acquisitions", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-concurrent-acquire-" }); - const results = yield* Effect.all( - Array.from({ length: 10 }, () => makeTestState(root).pipe(Effect.result)), - { concurrency: "unbounded" }, - ); - expect(results.every(Result.isSuccess)).toBe(true); - }), - ), - ); - // Windows has no directory fsync to fail. it.effect.skipIf(process.platform === "win32")( "fails publication when the directory fsync reports EIO", diff --git a/packages/stack/src/namespace/Publication.ts b/packages/stack/src/namespace/Publication.ts index e990b3334e..c1e2d57826 100644 --- a/packages/stack/src/namespace/Publication.ts +++ b/packages/stack/src/namespace/Publication.ts @@ -5,8 +5,6 @@ import * as Drivers from "./drivers/FileSystem.ts"; export interface PublishOptions { readonly target: string; readonly content: string; - /** "initial" publishes exclusively with a hard link; "replace" atomically renames over it. */ - readonly mode: "initial" | "replace"; readonly platform?: NodeJS.Platform; } @@ -14,11 +12,11 @@ let counter = 0; /** * Writes `content` as `target`, so a reader only ever sees the prior document or the complete new - * one. Stages the full content in `target`'s own directory and fsyncs it, publishes it with a - * hard link or a rename depending on `mode`, then fsyncs the directory. The staging path and its - * cleanup are reserved before anything is written to it, so a write that fails partway (for - * example `ENOSPC`) still leaves no file behind; a process killed mid publish leaves only the - * staging file, never a partially written target. + * one. Stages the full content in `target`'s own directory and fsyncs it, publishes it with an + * atomic rename, then fsyncs the directory. The staging path and its cleanup are reserved before + * anything is written to it, so a write that fails partway (for example `ENOSPC`) still leaves no + * file behind; a process killed mid publish leaves only the staging file, never a partially + * written target. */ export const publish = Effect.fn("Namespace.publish")(function* ( fs: FileSystem.FileSystem, @@ -41,18 +39,10 @@ export const publish = Effect.fn("Namespace.publish")(function* ( Effect.gen(function* () { yield* fs.writeFileString(staging, options.content, { mode: 0o600 }); yield* Drivers.fsyncFile(fs, staging); - if (options.mode === "initial") { - // Retried separately: once the link lands, the target exists, so retrying it together - // with removing the staging name would see a transient removal failure as `EEXIST` on - // the next attempt instead of as the sharing violation it actually was. - yield* retry(fs.link(staging, options.target)); - yield* retry(fs.remove(staging)).pipe(Effect.ignore); - } else { - yield* retry(fs.rename(staging, options.target)); - } + yield* retry(fs.rename(staging, options.target)); yield* Drivers.fsyncDirectory(fs, directory); }), - // Both modes already remove the staging name on success; only a failed attempt leaves it. + // The rename already clears the staging name on success; only a failed attempt leaves it. () => retry(fs.remove(staging, { force: true })).pipe(Effect.ignore), ).pipe(Effect.mapError((cause) => namespaceError("publish", cause))); }); diff --git a/packages/stack/src/namespace/Registry.ts b/packages/stack/src/namespace/Registry.ts index c740d60d25..710bc8f182 100644 --- a/packages/stack/src/namespace/Registry.ts +++ b/packages/stack/src/namespace/Registry.ts @@ -205,7 +205,6 @@ export const make = ( yield* Publication.publish(fs, path, { target: statePath(state.id), content: serialized, - mode: "replace", platform: options.platform, }); }); diff --git a/packages/stack/src/namespace/drivers/FileSystem.ts b/packages/stack/src/namespace/drivers/FileSystem.ts index de9f47bc83..6bd2be7172 100644 --- a/packages/stack/src/namespace/drivers/FileSystem.ts +++ b/packages/stack/src/namespace/drivers/FileSystem.ts @@ -1,4 +1,4 @@ -import { Effect, FileSystem, Path, Predicate } from "effect"; +import { Effect, FileSystem, Predicate } from "effect"; import type { PlatformError } from "effect/PlatformError"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no non-recursive directory removal operation. import { rmdir } from "node:fs/promises"; @@ -33,36 +33,3 @@ export const fsyncDirectory = ( process.platform === "win32" ? Effect.void : Effect.scoped(fs.open(directory, { flag: "r" }).pipe(Effect.flatMap((file) => file.sync))); - -let probeCounter = 0; - -/** - * Fails before any mutation when `directory`'s filesystem cannot hard-link within itself. - * Unsupported, by maintainer decision, rather than falling back to a non-atomic publish. The probe - * name is unique per call, so concurrent acquisitions under the same root never collide on it. - */ -export const assertHardLinkSupport = Effect.fn("Namespace.assertHardLinkSupport")(function* ( - fs: FileSystem.FileSystem, - path: Path.Path, - directory: string, -) { - const probe = path.join(directory, `.namespace-hardlink-probe-${process.pid}-${++probeCounter}`); - const probeLink = `${probe}.link`; - const cleanup = Effect.all( - [ - fs.remove(probe, { force: true }).pipe(Effect.ignore), - fs.remove(probeLink, { force: true }).pipe(Effect.ignore), - ], - { discard: true }, - ); - const supported = yield* fs.writeFileString(probe, "").pipe( - Effect.mapError((cause) => namespaceError("hardlink-support", cause)), - Effect.andThen(fs.link(probe, probeLink).pipe(Effect.isSuccess)), - Effect.ensuring(cleanup), - ); - if (!supported) - return yield* namespaceError( - "hardlink-support", - `${directory} is on a filesystem without hard-link support, which the stack namespace requires for atomic publication`, - ); -}); From e06e30cd1aa4874476e275416ce260f9e5a78cf4 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sat, 3 Oct 2026 06:38:58 +0200 Subject: [PATCH 009/142] test(stack): confirm owner exit before removing the SIGKILL scenario's temp root --- .../stack/src/namespace/Namespace.e2e.test.ts | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/packages/stack/src/namespace/Namespace.e2e.test.ts b/packages/stack/src/namespace/Namespace.e2e.test.ts index eef6c1d58b..1b16f1d8d0 100644 --- a/packages/stack/src/namespace/Namespace.e2e.test.ts +++ b/packages/stack/src/namespace/Namespace.e2e.test.ts @@ -63,26 +63,38 @@ it.live( Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; + // Scoped outside the owners: their scope below must fully close (confirming the second + // owner has actually exited, not just acknowledged shutdown) before this directory's own + // removal runs, or Windows reports EBUSY on a file the still-exiting owner has open. const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-e2e-sigkill-" }); const state = yield* makeTestState(root); const id = "stack"; yield* state.save(savedStack(root, id)); const options = { stateRoot: root, cacheRoot: root, stackId: id, entrypoint: ownerFixture }; - const first = yield* launchHost(state, options); - expect(yield* state.leased(id)).toBe(true); + yield* Effect.scoped( + Effect.gen(function* () { + const first = yield* launchHost(state, options); + expect(yield* state.leased(id)).toBe(true); - const released = yield* watchLeaseRelease(root, id); - yield* Effect.sync(() => process.kill(first.endpoint.pid, "SIGKILL")); - yield* released; - expect(yield* state.leased(id)).toBe(false); + const released = yield* watchLeaseRelease(root, id); + yield* Effect.sync(() => process.kill(first.endpoint.pid, "SIGKILL")); + yield* released; + expect(yield* state.leased(id)).toBe(false); - // No file is removed or repaired here; the kernel alone released the dead owner's lease. - const second = yield* Effect.acquireRelease(launchHost(state, options), (access) => - shutdownHost(access, true).pipe(Effect.ignore), + // No file is removed or repaired here; the kernel alone released the dead owner's lease. + const second = yield* Effect.acquireRelease(launchHost(state, options), (access) => + shutdownHost(access, true).pipe( + Effect.ignore, + Effect.andThen( + waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe(Effect.ignore), + ), + ), + ); + expect(second.endpoint.pid).not.toBe(first.endpoint.pid); + expect(yield* connectHost(state, id)).toEqual(second); + }), ); - expect(second.endpoint.pid).not.toBe(first.endpoint.pid); - expect(yield* connectHost(state, id)).toEqual(second); }), ).pipe(Effect.provide(testLayer)), 20_000, From e935859fab982c0346a41f52f48b0741960cf284 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sat, 3 Oct 2026 09:06:15 +0200 Subject: [PATCH 010/142] test(stack): stop test git from detaching background maintenance during cleanup --- packages/stack/tests/helpers/git.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/stack/tests/helpers/git.ts b/packages/stack/tests/helpers/git.ts index 439c946cab..cf4bb07714 100644 --- a/packages/stack/tests/helpers/git.ts +++ b/packages/stack/tests/helpers/git.ts @@ -20,6 +20,11 @@ const decode = (chunks: ReadonlyArray): string => { return new TextDecoder().decode(bytes); }; +/** Disables auto-gc/maintenance so Git never forks a detached writer into `.git/` after this + * command returns; test fixtures remove the repository right after these commands finish, and a + * detached `gc --auto`/`maintenance run --auto` would still be touching objects and packs then. */ +const NO_AUTO_MAINTENANCE = ["-c", "gc.auto=0", "-c", "maintenance.auto=false"]; + /** Runs Git while draining both output streams, preserving diagnostics on failure. */ export const runGit = ( cwd: string, @@ -28,7 +33,7 @@ export const runGit = ( Effect.scoped( Effect.gen(function* () { const command = `git ${args.join(" ")}`; - const child = yield* ChildProcess.make("git", [...args], { + const child = yield* ChildProcess.make("git", [...NO_AUTO_MAINTENANCE, ...args], { cwd, stdout: "pipe", stderr: "pipe", From 94f0c0f70b823ae85f90e028ca42a210811ecfa0 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sat, 3 Oct 2026 09:30:57 +0200 Subject: [PATCH 011/142] fix(stack): wait for container loopback publication instead of reading it once --- packages/stack/src/runtime/Container.ts | 91 +++++++++++++++++++------ 1 file changed, 72 insertions(+), 19 deletions(-) diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index cc0b192d55..d9325b6e0e 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -233,6 +233,9 @@ const pullBackoff = Schedule.exponential("2 seconds").pipe(Schedule.jittered); /** `docker create` only writes metadata; a healthy daemon answers well within this bound. */ const CREATE_TIMEOUT: Duration.Input = "2 minutes"; +/** A started container's loopback publish can lag under concurrent engine load before landing. */ +const PORT_PUBLISH_TIMEOUT: Duration.Input = "2 minutes"; + const PublishedPorts = Schema.Record( Schema.String, Schema.NullOr( @@ -245,6 +248,13 @@ const PublishedPorts = Schema.Record( ), ); +/** The published ports together with the run state they were observed in. */ +const PublicationState = Schema.Struct({ + Ports: Schema.NullOr(PublishedPorts), + Status: Schema.String, + ExitCode: Schema.Finite, +}); + const mountField = (key: string, value: string) => { const field = `${key}=${value}`; return /[,"\n\r]/u.test(field) ? `"${field.replaceAll('"', '""')}"` : field; @@ -759,25 +769,68 @@ export const makeContainerRuntime = (options: { getWaiter, ).pipe(Effect.flatMap((fiber) => Fiber.join(fiber))); owned = { ...partial, exitCode }; - const text = yield* run([ - "inspect", - "--format", - "{{json .NetworkSettings.Ports}}", - name, - ]); - const bindings = yield* Schema.decodeEffect(Schema.fromJsonString(PublishedPorts))( - text, - ).pipe(Effect.mapError((cause) => errorFor("inspect", cause))); - const ports: Record = {}; - for (const port of spec.ports ?? []) { - const value = bindings[`${port}/tcp`]?.find( - (binding) => binding.HostIp === "127.0.0.1", - )?.HostPort; - const actual = Number(value); - if (!Number.isInteger(actual) || actual < 1 || actual > 65535) - return yield* errorFor("inspect", `No loopback publication for ${port}`); - ports[port] = actual; - } + const requestedPorts = spec.ports ?? []; + const inspectPublished = Effect.gen(function* () { + const text = yield* run([ + "inspect", + "--format", + '{"Ports":{{json .NetworkSettings.Ports}},"Status":{{json .State.Status}},"ExitCode":{{.State.ExitCode}}}', + name, + ]); + const state = yield* Schema.decodeEffect(Schema.fromJsonString(PublicationState))( + text, + ).pipe(Effect.mapError((cause) => errorFor("inspect", cause))); + const bindings = state.Ports ?? {}; + const published: Record = {}; + const pending: Array = []; + for (const port of requestedPorts) { + const value = bindings[`${port}/tcp`]?.find( + (binding) => binding.HostIp === "127.0.0.1", + )?.HostPort; + const actual = Number(value); + if (Number.isInteger(actual) && actual >= 1 && actual <= 65535) + published[port] = actual; + else pending.push(port); + } + if (pending.length > 0 && state.Status !== "running") + return yield* errorFor( + "inspect", + `Container ${state.Status} (exit code ${state.ExitCode}) before publishing ${pending.join(", ")}`, + ); + return { published, pending }; + }); + // The engine can report a container started before its loopback publish lands, + // so poll for it instead of trusting a single inspect under load. + const awaitPublishedPorts = Effect.fn("Container.awaitPublishedPorts")(function* () { + const observed = yield* Ref.make<{ + readonly published: Record; + readonly pending: ReadonlyArray; + }>({ published: {}, pending: requestedPorts }); + yield* withAttemptCount( + inspectPublished.pipe(Effect.tap((result) => Ref.set(observed, result))), + (counted) => + counted.pipe( + Effect.repeat({ + schedule: Schedule.spaced("250 millis"), + while: (result) => result.pending.length > 0, + }), + ), + ).pipe( + Effect.timeout(PORT_PUBLISH_TIMEOUT), + Effect.catchTag("TimeoutError", () => + Ref.get(observed).pipe( + Effect.flatMap(({ pending }) => + errorFor( + "inspect", + `No loopback publication for ${pending.join(", ")} after ${PORT_PUBLISH_TIMEOUT}`, + ), + ), + ), + ), + ); + return (yield* Ref.get(observed)).published; + }); + const ports = yield* awaitPublishedPorts(); const logProcess = attached ?? (yield* Effect.gen(function* () { From 85e99f6b6579b24f867bbc328ad380a666494a0c Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sat, 3 Oct 2026 09:45:36 +0200 Subject: [PATCH 012/142] test(stack): give the real-engine removal wait the container test budget --- .../src/runtime/Container.integration.test.ts | 77 ++++++++++--------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/packages/stack/src/runtime/Container.integration.test.ts b/packages/stack/src/runtime/Container.integration.test.ts index 9028e812db..413f859be4 100644 --- a/packages/stack/src/runtime/Container.integration.test.ts +++ b/packages/stack/src/runtime/Container.integration.test.ts @@ -961,43 +961,46 @@ describe("container process adapter", () => { }).pipe(Effect.provide(NodeServices.layer)), ); - it.live("waits for a real removal observed as removing", () => - Effect.gen(function* () { - const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; - const crypto = yield* Crypto.Crypto; - const token = yield* crypto.randomUUIDv4; - const instanceId = `pending-remove-${token}`; - const pendingProbes = yield* Ref.make(2); - const lostResult = yield* Ref.make(false); - const spawner = makePendingRemoveSpawner(delegate, pendingProbes, lostResult); - yield* Effect.ensuring( - Effect.gen(function* () { - const result = yield* Effect.scoped( - Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); - yield* runtime.prepare(image); - const process = yield* runtime.launch({ - image, - stackId: "n".repeat(64), - instanceId, - env: {}, - args: ["-e", stoppableIdleScript], - }); - yield* process.stop; - yield* process.remove; - }), - ).pipe( - Effect.exit, - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - ); - expect(yield* Ref.get(lostResult)).toBe(true); - expect(yield* Ref.get(pendingProbes)).toBe(0); - expect(Exit.isSuccess(result)).toBe(true); - expect(yield* idsByInstance(instanceId)).toHaveLength(0); - }), - removeByInstance(instanceId).pipe(Effect.orDie), - ); - }).pipe(Effect.provide(NodeServices.layer)), + it.live( + "waits for a real removal observed as removing", + () => + Effect.gen(function* () { + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const crypto = yield* Crypto.Crypto; + const token = yield* crypto.randomUUIDv4; + const instanceId = `pending-remove-${token}`; + const pendingProbes = yield* Ref.make(2); + const lostResult = yield* Ref.make(false); + const spawner = makePendingRemoveSpawner(delegate, pendingProbes, lostResult); + yield* Effect.ensuring( + Effect.gen(function* () { + const result = yield* Effect.scoped( + Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + yield* runtime.prepare(image); + const process = yield* runtime.launch({ + image, + stackId: "n".repeat(64), + instanceId, + env: {}, + args: ["-e", stoppableIdleScript], + }); + yield* process.stop; + yield* process.remove; + }), + ).pipe( + Effect.exit, + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + expect(yield* Ref.get(lostResult)).toBe(true); + expect(yield* Ref.get(pendingProbes)).toBe(0); + expect(Exit.isSuccess(result)).toBe(true); + expect(yield* idsByInstance(instanceId)).toHaveLength(0); + }), + removeByInstance(instanceId).pipe(Effect.orDie), + ); + }).pipe(Effect.provide(NodeServices.layer)), + { timeout: 120_000 }, ); it.live("retains the remove failure when removing never reaches absence", () => From 380adeec338028c9657137bdabad8db82b0e7f70 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sat, 3 Oct 2026 15:41:32 +0200 Subject: [PATCH 013/142] refactor(stack): own resource claims, paths, environments and the engine target --- .github/workflows/test.yml | 1 + apps/cli/docs/analytics.md | 2 +- apps/cli/docs/stack-commands.md | 14 +- apps/cli/src/command-internal/stack-api.ts | 13 +- .../command-internal/stack-local-database.ts | 2 +- .../cli/src/command-internal/stack-runtime.ts | 20 +- .../stack-runtime.unit.test.ts | 6 +- .../stack-shadow.integration.test.ts | 6 +- apps/cli/src/command-internal/stack-shadow.ts | 10 +- ...elta-next-shadow.stack.integration.test.ts | 4 +- .../stack/destroy/SIDE_EFFECTS.md | 23 +- .../stack/destroy/destroy.handler.ts | 8 +- ...oy.runtime-unavailable.integration.test.ts | 69 +-- .../experimental/stack/list/SIDE_EFFECTS.md | 4 +- .../stack/prepare/SIDE_EFFECTS.md | 2 +- .../stack/prepare/prepare.command.ts | 4 +- .../stack/prepare/prepare.integration.test.ts | 54 +- .../stack/stack.shared.integration.test.ts | 13 +- .../experimental/stack/start/SIDE_EFFECTS.md | 10 +- .../experimental/stack/start/start.command.ts | 6 +- .../experimental/stack/start/start.handler.ts | 3 +- .../stack/start/start.integration.test.ts | 45 +- .../stack/status/status.handler.ts | 4 +- .../tests/helpers/child-process-spawner.ts | 11 +- packages/stack/README.md | 8 +- .../stack/src/Commands.integration.test.ts | 28 +- packages/stack/src/Network.ts | 9 +- packages/stack/src/Owner.ts | 108 +++- ...ost.container-shutdown.integration.test.ts | 52 +- ....container-sweep-retry.integration.test.ts | 516 +++++++++++++++--- .../stack/src/StackHost.integration.test.ts | 2 + packages/stack/src/StackHost.ts | 41 +- packages/stack/src/StackNamespace.ts | 9 +- packages/stack/src/Sweep.integration.test.ts | 15 +- packages/stack/src/Sweep.ts | 13 +- .../create.owner-rollback.integration.test.ts | 61 ++- ...oy.runtime-unavailable.integration.test.ts | 241 ++++---- packages/stack/src/effect.integration.test.ts | 23 +- packages/stack/src/effect.ts | 116 +--- .../stack/src/functions/FunctionsBootstrap.ts | 31 +- .../functions-bootstrap.integration.test.ts | 49 -- ...dRunner.initialization.integration.test.ts | 2 + ...dRunner.native-cleanup.integration.test.ts | 2 + packages/stack/src/host/CommandRunner.ts | 35 +- packages/stack/src/internal/test-run-label.ts | 2 +- packages/stack/src/namespace/Claims.ts | 177 ++++++ .../namespace/Environment.integration.test.ts | 52 ++ packages/stack/src/namespace/Environment.ts | 104 ++++ .../stack/src/namespace/Namespace.e2e.test.ts | 100 +++- .../src/namespace/Paths.integration.test.ts | 223 ++++++++ packages/stack/src/namespace/Paths.ts | 196 +++++++ packages/stack/src/namespace/Registry.ts | 13 +- .../stack/src/namespace/drivers/FileSystem.ts | 48 +- .../src/runtime/Container.integration.test.ts | 437 +++++++++++++-- packages/stack/src/runtime/Container.ts | 369 ++++++++----- packages/stack/src/runtime/NativeProcess.ts | 11 +- ...container-image-mirror.integration.test.ts | 59 +- .../native-process.integration.test.ts | 28 +- .../runtime/postgres-user.integration.test.ts | 147 +++++ packages/stack/src/runtime/postgres-user.ts | 179 +++++- ...nalyticsVectorImgproxy.integration.test.ts | 17 +- .../services/AuthStorage.integration.test.ts | 12 +- .../src/services/Catalog.integration.test.ts | 3 + packages/stack/src/services/Catalog.ts | 24 +- .../src/services/Database.integration.test.ts | 27 + packages/stack/src/services/Database.ts | 292 +++++----- .../DatabaseSnapshot.integration.test.ts | 19 +- .../stack/src/services/DatabaseSnapshot.ts | 5 +- .../services/Functions.integration.test.ts | 105 +++- packages/stack/src/services/Functions.ts | 19 +- packages/stack/src/services/Imgproxy.ts | 2 + .../services/InstanceRoot.integration.test.ts | 149 ----- packages/stack/src/services/InstanceRoot.ts | 155 ------ .../src/services/Mail.integration.test.ts | 5 + .../src/services/Pooler.integration.test.ts | 5 + .../ProcessRecipe.integration.test.ts | 61 ++- packages/stack/src/services/ProcessRecipe.ts | 58 +- .../src/services/Realtime.integration.test.ts | 5 + packages/stack/src/services/Recipe.ts | 11 +- .../src/services/Rest.integration.test.ts | 5 + packages/stack/src/services/Storage.ts | 1 + .../StorageGateway.integration.test.ts | 16 +- .../src/services/Studio.integration.test.ts | 43 ++ packages/stack/src/services/Studio.ts | 4 + .../src/services/Vector.integration.test.ts | 23 +- packages/stack/src/services/Vector.ts | 128 +---- .../DockerDatabaseStorage.integration.test.ts | 432 +++++++++++++-- .../src/storage/DockerDatabaseStorage.ts | 151 ++--- .../stack/src/testing.integration.test.ts | 24 +- packages/stack/src/testing.ts | 2 +- .../whole-stack.docker.lifecycle.e2e.test.ts | 7 +- .../whole-stack.native.lifecycle.e2e.test.ts | 7 +- packages/stack/tests/claims.ts | 15 + .../stack/tests/compiled-dispatch-fixture.ts | 13 + .../tests/docker-fixture.integration.test.ts | 5 + .../docker-storage-publish-loop-fixture.ts | 64 +++ packages/stack/tests/engine-target.ts | 13 + packages/stack/tests/owner-rpc.ts | 2 + .../stack/tests/whole-stack/confinement.ts | 162 ++++++ packages/stack/tests/whole-stack/scenarios.ts | 96 +++- 100 files changed, 4292 insertions(+), 1735 deletions(-) create mode 100644 packages/stack/src/namespace/Claims.ts create mode 100644 packages/stack/src/namespace/Environment.integration.test.ts create mode 100644 packages/stack/src/namespace/Environment.ts create mode 100644 packages/stack/src/namespace/Paths.integration.test.ts create mode 100644 packages/stack/src/namespace/Paths.ts create mode 100644 packages/stack/src/runtime/postgres-user.integration.test.ts delete mode 100644 packages/stack/src/services/InstanceRoot.integration.test.ts delete mode 100644 packages/stack/src/services/InstanceRoot.ts create mode 100644 packages/stack/src/services/Studio.integration.test.ts create mode 100644 packages/stack/tests/claims.ts create mode 100644 packages/stack/tests/docker-storage-publish-loop-fixture.ts create mode 100644 packages/stack/tests/engine-target.ts create mode 100644 packages/stack/tests/whole-stack/confinement.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e2b69680d5..0184dd99e9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -162,6 +162,7 @@ jobs: src/Ports.integration.test.ts src/storage/DirectoryCopy.integration.test.ts src/namespace/Namespace.integration.test.ts + src/namespace/Paths.integration.test.ts src/HostProcess.integration.test.ts src/Network.integration.test.ts src/Proxy.integration.test.ts diff --git a/apps/cli/docs/analytics.md b/apps/cli/docs/analytics.md index 225713f0fc..61472e0b30 100644 --- a/apps/cli/docs/analytics.md +++ b/apps/cli/docs/analytics.md @@ -90,7 +90,7 @@ Local-runtime commands also carry properties that split error rates by runtime: - `stack_backend` (`legacy` or `stack`) on commands routed by `experimental.stack`; see [backend selection](stack-commands.md) -- `stack_runtime` (`docker`, `podman`, or `native`): the runtime of the stack a stack-backend +- `stack_runtime` (`docker` or `native`): the runtime of the stack a stack-backend command looked up, which for `--db-url` may be the project stack even when the URL targets another database - `orioledb` (boolean) once the command resolves local database config, `true` when the project diff --git a/apps/cli/docs/stack-commands.md b/apps/cli/docs/stack-commands.md index dd8cbdd896..e443c8d669 100644 --- a/apps/cli/docs/stack-commands.md +++ b/apps/cli/docs/stack-commands.md @@ -2,8 +2,8 @@ `supabase stack` manages local stacks with the new experimental runtime. It is unstable, its command interface may change, and it is excluded from the CLI compatibility promise. It is -available when the `experimental.stack` feature flag is enabled and supports Docker, Podman, and -native runtimes. +available when the `experimental.stack` feature flag is enabled and supports Docker and native +runtimes. Native PostgreSQL requires passwords for every role except `supabase_admin`. Its local bootstrap and password-reconciliation connection uses that administrative role, so a native @@ -22,11 +22,11 @@ and password-reconciliation connection uses that administrative role, so a nativ Use each command's `--help` for its available targeting and runtime options. -A new stack created with `--runtime auto` uses Docker when its daemon answers, otherwise Podman -when its engine answers, and otherwise native on Linux x64/arm64 and macOS arm64. On other -platforms without a reachable engine, the command fails and asks you to start Docker or Podman. The -selected runtime is saved with the stack and reused without probing; when auto selection skips -Docker, the command prints a notice saying so. To switch, destroy the stack or choose a different +A new stack created with `--runtime auto` uses Docker when its daemon answers, and otherwise native +on Linux x64/arm64 and macOS arm64. On other platforms without a reachable engine, the command +fails and asks you to start Docker. The selected runtime is saved with the stack and reused without +probing; when auto selection skips Docker, the command prints a notice saying so. To switch, +destroy the stack or choose a different `--stack` name. When an explicit `--runtime docker` or a saved Docker stack cannot reach Docker, the failure asks you to install or start it, and also suggests `--runtime native` for a new stack on platforms that support native. Project stacks created by database commands, and shadow stacks diff --git a/apps/cli/src/command-internal/stack-api.ts b/apps/cli/src/command-internal/stack-api.ts index 4c09eff029..4c52291daf 100644 --- a/apps/cli/src/command-internal/stack-api.ts +++ b/apps/cli/src/command-internal/stack-api.ts @@ -67,16 +67,11 @@ export const stackApiLayer = Layer.effect( }), ).pipe(Layer.provide(FetchHttpClient.layer)); -/** Describes the engine resources a destroy left behind and the commands that remove them. */ +/** Describes why a destroy left engine resources behind, without promising automatic cleanup. */ export const skippedRuntimeCleanupWarning = ( subject: string, - result: Extract, -): string => { - const engineName = result.engine === "docker" ? "Docker" : "Podman"; - return [ - `${engineName} was unavailable, so ${engineName} resources for ${subject} were not removed. Once it is running, remove them with:`, - ...result.cleanupCommands.map((command) => ` ${command}`), - ].join("\n"); -}; + _result: Extract, +): string => + `Docker was unavailable, so Docker resources for ${subject} were not removed. Restore Docker and run "supabase stack destroy" again to finish removing it.`; export type { Stack }; diff --git a/apps/cli/src/command-internal/stack-local-database.ts b/apps/cli/src/command-internal/stack-local-database.ts index 613bb47d10..8ecda0e7b8 100644 --- a/apps/cli/src/command-internal/stack-local-database.ts +++ b/apps/cli/src/command-internal/stack-local-database.ts @@ -127,7 +127,7 @@ const RUNTIME_UNAVAILABLE = new StackRuntimeUnavailableError({ }); export const stackRequireProjectRuntime: Effect.Effect< - "native" | "docker" | "podman", + "native" | "docker", StackRuntimeUnavailableError, CommandSettings | StackApi | Path.Path > = stackProjectRuntime.pipe( diff --git a/apps/cli/src/command-internal/stack-runtime.ts b/apps/cli/src/command-internal/stack-runtime.ts index aa7fb4dcf3..1e4ee9c1db 100644 --- a/apps/cli/src/command-internal/stack-runtime.ts +++ b/apps/cli/src/command-internal/stack-runtime.ts @@ -13,7 +13,7 @@ import { withChildTraceEnv, withProcessSpan } from "../shared/telemetry/spans.ts import { recordCommandTelemetry } from "../telemetry/command-telemetry-attributes.ts"; /** Runtime that executes a local stack's services. */ -export type StackRuntime = "native" | "docker" | "podman"; +export type StackRuntime = "native" | "docker"; /** Records the runtime of the stack the enclosing command operates on. */ export const recordStackRuntimeTelemetry = (runtime: StackRuntime) => @@ -32,13 +32,12 @@ export class StackRuntimeSelectionError extends Data.TaggedError("StackRuntimeSe } } -/** A cold Docker Desktop or Podman machine can take several seconds to answer its first request. */ +/** A cold Docker Desktop machine can take several seconds to answer its first request. */ const PROBE_TIMEOUT = "10 seconds"; -/** Each probe exits non-zero unless the engine's daemon or service answers. */ +/** Each probe exits non-zero unless the engine's daemon answers. */ const engineProbes = [ { runtime: "docker", args: ["version", "--format", "{{.Server.Version}}"] }, - { runtime: "podman", args: ["info", "--format", "{{.Version.Version}}"] }, ] as const; const engineReachable = ( @@ -80,12 +79,9 @@ export const automaticRuntimeNotice = ( ): string | undefined => requested !== undefined || selected === "docker" ? undefined - : `Docker didn't answer, so this new stack uses the ${selected === "podman" ? "Podman" : "native"} runtime, which is saved with the stack. To use Docker, start it, then run \`supabase stack destroy\` and start again, or choose a different --stack name with --runtime docker.`; + : `Docker didn't answer, so this new stack uses the native runtime, which is saved with the stack. To use Docker, start it, then run \`supabase stack destroy\` and start again, or choose a different --stack name with --runtime docker.`; -/** - * Returns the requested or saved runtime unchanged; otherwise the first of Docker, Podman, or - * native that is usable on this host. - */ +/** Returns the requested or saved runtime unchanged; otherwise Docker, or native, if usable. */ export const selectStackRuntime = Effect.fn("StackRuntime.select")(function* ( requested: StackRuntime | undefined, ) { @@ -98,7 +94,7 @@ export const selectStackRuntime = Effect.fn("StackRuntime.select")(function* ( reason: "native-unsupported", message: `Native artifacts are unsupported on ${platform}/${arch}.`, suggestion: - "Start Docker or Podman and rerun with --runtime docker or --runtime podman; if this stack already exists as native, run supabase stack destroy first.", + "Start Docker and rerun with --runtime docker; if this stack already exists as native, run supabase stack destroy first.", }); } return requested; @@ -111,7 +107,7 @@ export const selectStackRuntime = Effect.fn("StackRuntime.select")(function* ( if (defaultRuntime({ os: platform, arch }) === "native") return "native"; return yield* new StackRuntimeSelectionError({ reason: "engine-unreachable", - message: `Neither Docker nor Podman is reachable, and native stacks are not supported on ${platform}/${arch}.`, - suggestion: "Start Docker or Podman, then rerun the command.", + message: `Docker is not reachable, and native stacks are not supported on ${platform}/${arch}.`, + suggestion: "Start Docker, then rerun the command.", }); }, Effect.tap(recordStackRuntimeTelemetry)); diff --git a/apps/cli/src/command-internal/stack-runtime.unit.test.ts b/apps/cli/src/command-internal/stack-runtime.unit.test.ts index 79794d8306..7b44291715 100644 --- a/apps/cli/src/command-internal/stack-runtime.unit.test.ts +++ b/apps/cli/src/command-internal/stack-runtime.unit.test.ts @@ -13,14 +13,14 @@ describe("selectStackRuntime", () => { Effect.gen(function* () { const recorded = yield* Ref.make({}); - const runtime = yield* selectStackRuntime("podman").pipe( + const runtime = yield* selectStackRuntime("docker").pipe( Effect.provideService(CommandTelemetryAttributes, { record: (values) => Ref.update(recorded, (current) => ({ ...current, ...values })), }), ); - expect(runtime).toBe("podman"); - expect((yield* Ref.get(recorded)).stack_runtime).toBe("podman"); + expect(runtime).toBe("docker"); + expect((yield* Ref.get(recorded)).stack_runtime).toBe("docker"); }).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, runtimeInfoLayer))), ); }); diff --git a/apps/cli/src/command-internal/stack-shadow.integration.test.ts b/apps/cli/src/command-internal/stack-shadow.integration.test.ts index 40b038279b..05a0b08ead 100644 --- a/apps/cli/src/command-internal/stack-shadow.integration.test.ts +++ b/apps/cli/src/command-internal/stack-shadow.integration.test.ts @@ -458,14 +458,13 @@ describe("stack shadow databases", () => { ); it.live( - "prints the cleanup commands when a shadow's destroy skips its engine", + "warns that a shadow's engine cleanup is pending when destroy skips its engine", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-shadow-skipped-" }); const output = mockOutput(); - const cleanupCommand = "docker rm --force $(docker ps --all --quiet)"; // A real registration whose database creation fails fast and whose destroy reports skipped cleanup. const api = Layer.effect( StackApi, @@ -485,7 +484,6 @@ describe("stack shadow databases", () => { destroy: Effect.succeed({ runtimeCleanup: "skipped", engine: "docker", - cleanupCommands: [cleanupCommand], } as const), })), ), @@ -510,7 +508,7 @@ describe("stack shadow databases", () => { ); expect(output.stderrText).toMatch( - /Warning: Docker was unavailable, so Docker resources for shadow stack [0-9a-f]{64} were not removed\. Once it is running, remove them with:\n {2}docker rm --force \$\(docker ps --all --quiet\)\n/u, + /Warning: Docker was unavailable, so Docker resources for shadow stack [0-9a-f]{64} were not removed\. Restore Docker and run "supabase stack destroy" again to finish removing it\.\n/u, ); }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), 120_000, diff --git a/apps/cli/src/command-internal/stack-shadow.ts b/apps/cli/src/command-internal/stack-shadow.ts index 86c2ee6276..c6f91e8063 100644 --- a/apps/cli/src/command-internal/stack-shadow.ts +++ b/apps/cli/src/command-internal/stack-shadow.ts @@ -6,7 +6,7 @@ import { RuntimeInfo } from "../shared/runtime/runtime-info.service.ts"; import { skippedRuntimeCleanupWarning, StackApi } from "./stack-api.ts"; import { initializeStackDatabase } from "./stack-bootstrap.ts"; import { stackProjectRuntime } from "./stack-local-database.ts"; -import { selectStackRuntime } from "./stack-runtime.ts"; +import { selectStackRuntime, type StackRuntime } from "./stack-runtime.ts"; import { parseConnectionString } from "./db-config.parse.ts"; import { toPostgresURL } from "./postgres-url.ts"; import { @@ -19,15 +19,13 @@ import { listLocalMigrationPaths } from "./migration-history.ts"; import { applyMigrations } from "./migration-apply.ts"; import { stackShadowCacheEntry, stackShadowCacheRoles } from "./stack-shadow-cache.ts"; -type Runtime = "native" | "docker" | "podman"; - export interface StackShadowAcquiredHandle { readonly stack: Stack; readonly database: DatabaseInstance; readonly url: string; readonly host: string; readonly port: number; - readonly runtime: Runtime; + readonly runtime: StackRuntime; readonly version: string; /** Cache lineage when a baseline was restored or atomically published by this handle. */ readonly snapshotKey?: string; @@ -36,7 +34,7 @@ export interface StackShadowAcquiredHandle { interface ShadowOptions { readonly port?: number; - readonly runtime?: Runtime; + readonly runtime?: StackRuntime; readonly webhooks?: SetupDatabaseOptions["webhooks"]; readonly bypassCache?: boolean; } @@ -81,7 +79,7 @@ const acquireNamespace = Effect.fn("StackShadow.acquireNamespace")(function* (op const initialize = Effect.fn("StackShadow.initialize")(function* ( stack: Stack, - runtime: Runtime, + runtime: StackRuntime, input: ShadowSetupInput, opts: ShadowOptions, ) { diff --git a/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts b/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts index 9b4a0acf11..2ee1e255ab 100644 --- a/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts +++ b/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts @@ -92,7 +92,7 @@ describe("pg-delta next stack shadow provisioning", () => { ); const stateRoot = `${root}/stacks`; - const engines = containerEngineSpawner({ docker: "missing", podman: "missing" }); + const engines = containerEngineSpawner({ docker: "missing" }); const settings = mockCommandSettings({ workdir: root, supabaseHome: root }); const output = mockOutput().layer; const apiLayer = stackApiLayer.pipe( @@ -155,7 +155,7 @@ describe("pg-delta next stack shadow provisioning", () => { ).toEqual([{ table_name: null }]); const api = yield* StackApi; expect(yield* api.discover({ stateRoot })).toHaveLength(2); - expect(engines.spawned.map(({ command }) => command)).toEqual(["docker", "podman"]); + expect(engines.spawned.map(({ command }) => command)).toEqual(["docker"]); return plan; }).pipe(Effect.provide(services)), ); diff --git a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md index 8f7f722738..15e3f69b12 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md @@ -24,14 +24,13 @@ A failed destroy may leave its owner running; retry destruction or use `stack stop` to shut down that owner. When no owner is running and the engine reports that its daemon cannot be -reached, destruction removes the local namespace and host data anyway, warns on -stderr that the stack's engine resources were not removed, and exits 0. The -warning lists the commands that remove them once the engine is running: one for -the stack's containers, and one per database whose data is kept in an engine -volume. Any other engine failure, an owner starting during destruction, or -host data the current user cannot delete (such as database files a container -wrote as its own user), fails the command before anything is removed and keeps -the stack registered; the last case asks to start the engine and retry. +reached, destruction removes nothing: the registration, port claims and host +data stay as they are, stderr warns that Docker resources were not removed, and +the command exits 0 reporting the stack as not destroyed. Run destroy again once +Docker is reachable. Any other engine failure, an owner starting during +destruction, or resource claims the owner cannot reconcile (for example a +container on a different daemon) fails the command and keeps the stack +registered; the error lists the remaining claims and the `claims.json` path. ## Files and network @@ -45,10 +44,10 @@ routing/settings can read project configuration and profiles. ## Output, exit codes and telemetry Text prints `Stack destroyed.`, or, when engine cleanup was skipped, -`Stack was removed locally; its resources remain until the commands above are run.` -JSON and stream-json success data contain `destroyed: true`, `id`, and -`runtimeCleanup` (`complete` or `skipped`); a skipped cleanup also carries `engine` and -`cleanupCommands`. Exit 0 on destruction, 1 on invalid flags, missing +`Stack could not be fully destroyed because Docker is unreachable; restore it and run "supabase stack destroy" again.` +JSON and stream-json success data contain `destroyed` (`false` when engine +cleanup was skipped), `id`, and `runtimeCleanup` (`complete` or `skipped`); a +skipped cleanup also carries `engine`. Exit 0 on destruction or skipped engine cleanup, 1 on invalid flags, missing selection, rejected/cancelled confirmation or cleanup failure, and 130 on interruption. Standard command telemetry is unchanged, with no custom events. Telemetry flushes on success and failure to diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts index 07cde13aee..b26f30bc22 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts @@ -114,12 +114,16 @@ export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( if (result.runtimeCleanup === "skipped") yield* output.warn(skippedRuntimeCleanupWarning(`stack ${target.id}`, result)); if (output.format !== "text") - yield* output.success("", { destroyed: true, id: target.id, ...result }); + yield* output.success("", { + destroyed: result.runtimeCleanup === "complete", + id: target.id, + ...result, + }); else if (result.runtimeCleanup === "complete") yield* output.raw(`Stack ${target.id} destroyed.\n`); else yield* output.raw( - `Stack ${target.id} was removed locally; its ${result.engine === "docker" ? "Docker" : "Podman"} resources remain until the commands above are run.\n`, + `Stack ${target.id} could not be fully destroyed because Docker is unreachable; restore it and run "supabase stack destroy" again.\n`, ); }); return yield* body.pipe(Effect.ensuring(telemetryState.flush)); diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts index 625b9339f4..68c8ff8c01 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts @@ -13,42 +13,42 @@ import { stackDestroy } from "./destroy.handler.ts"; const live = Layer.provideMerge(stackApiLayer, BunServices.layer); +/** + * A registration real enough for target resolution, whose `destroy` reports the skipped-engine + * outcome directly: the namespace reconcile this models (claim stays, a later acquisition + * finishes it) is covered by packages/stack's own suite; this fixture is only the CLI's + * formatting of that result. + */ const fixture = Effect.fn("StackDestroyRuntimeUnavailableTest.fixture")(function* ( format: "text" | "json", ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-runtime-" }); - const binDir = path.join(root, "bin"); - yield* fs.makeDirectory(binDir, { recursive: true }); - const dockerShim = path.join(binDir, "docker"); - yield* fs.writeFileString( - dockerShim, - "#!/bin/sh\necho 'Cannot connect to the Docker daemon at tcp://127.0.0.1:1. Is the docker daemon running?' >&2\nexit 1\n", - ); - yield* fs.chmod(dockerShim, 0o755); - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the detached host subprocess inherits PATH; this is not application config. - const originalPath = process.env.PATH; - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. - process.env.PATH = `${binDir}:${originalPath ?? ""}`; - yield* Effect.addFinalizer(() => - Effect.sync(() => { - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. - process.env.PATH = originalPath; - }), - ); - - const api = yield* StackApi; + const realApi = yield* StackApi; const locations = { stateRoot: path.join(root, "stacks"), cacheRoot: path.join(root, "cache") }; - const stack = yield* api.create({ ...locations, projectRoot: root, runtime: "docker" }); + const stack = yield* realApi.create({ ...locations, projectRoot: root, runtime: "docker" }); + const api = StackApi.of({ + ...realApi, + open: (options) => + realApi.open(options).pipe( + Effect.map((opened) => ({ + ...opened, + destroy: Effect.succeed({ runtimeCleanup: "skipped", engine: "docker" } as const), + })), + ), + }); const output = mockOutput({ interactive: false, format }); const telemetry = mockTelemetryStateTracked(); const settings = mockCommandSettings({ workdir: root, supabaseHome: root }); const layer = Layer.mergeAll( + Layer.succeed(StackApi, api), output.layer, telemetry.layer, settings, - stackTargetResolverLayer.pipe(Layer.provide(settings)), + stackTargetResolverLayer.pipe( + Layer.provide(Layer.mergeAll(Layer.succeed(StackApi, api), settings)), + ), mockTty({ stdinIsTty: false }), mockStdin(false), Layer.succeed(YesFlag, true), @@ -64,48 +64,37 @@ const fixture = Effect.fn("StackDestroyRuntimeUnavailableTest.fixture")(function }; }); -const containerCleanup = (id: string) => - `sh -c 'ids=\\$\\(docker ps --all --quiet --no-trunc --filter '\\\\''label=com\\.supabase\\.stack=${id}'\\\\'' --filter '\\\\''label=com\\.supabase\\.stack-root=[^']+/${id}/data'\\\\''\\) && \\{ \\[ -z "\\$ids" \\] \\|\\| docker rm --force \\$ids; \\}'`; - it.live( - "removes a stack locally and lists its cleanup commands when its engine is unreachable", + "keeps a stack registered and reports pending engine cleanup when its engine is unreachable", () => Effect.gen(function* () { const f = yield* fixture("text"); yield* stackDestroy(f.flags).pipe(Effect.provide(f.layer)); expect(f.output.stdoutText).toBe( - `Stack ${f.stack.id} was removed locally; its Docker resources remain until the commands above are run.\n`, + `Stack ${f.stack.id} could not be fully destroyed because Docker is unreachable; restore it and run "supabase stack destroy" again.\n`, ); expect(f.output.messages).toContainEqual({ type: "warn", - message: expect.stringMatching( - new RegExp( - `^Docker was unavailable, so Docker resources for stack ${f.stack.id} were not removed\\. Once it is running, remove them with:\\n ${containerCleanup(f.stack.id)}$`, - "u", - ), - ), + message: `Docker was unavailable, so Docker resources for stack ${f.stack.id} were not removed. Restore Docker and run "supabase stack destroy" again to finish removing it.`, }); - expect(yield* f.api.discover(f.locations)).toEqual([]); + expect(yield* f.api.discover(f.locations)).toHaveLength(1); }).pipe(Effect.scoped, Effect.provide(live)), ); -it.live("reports skipped engine cleanup and its commands in the JSON result", () => +it.live("reports skipped engine cleanup in the JSON result", () => Effect.gen(function* () { const f = yield* fixture("json"); yield* stackDestroy(f.flags).pipe(Effect.provide(f.layer)); expect(f.output.messages).toContainEqual( expect.objectContaining({ data: { - destroyed: true, + destroyed: false, id: f.stack.id, runtimeCleanup: "skipped", engine: "docker", - cleanupCommands: [ - expect.stringMatching(new RegExp(`^${containerCleanup(f.stack.id)}$`, "u")), - ], }, }), ); - expect(yield* f.api.discover(f.locations)).toEqual([]); + expect(yield* f.api.discover(f.locations)).toHaveLength(1); }).pipe(Effect.scoped, Effect.provide(live)), ); diff --git a/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md index d4ada831f3..c70d13f8f4 100644 --- a/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md @@ -15,8 +15,8 @@ no stack state files are written. Telemetry flushes to ## Output -Entries contain `id`, `project_root`, `name`, `branch_context`, `runtime` (native, -docker, or podman), and `owner` (reachable or unavailable). Owner availability is +Entries contain `id`, `project_root`, `name`, `branch_context`, `runtime` (native +or docker), and `owner` (reachable or unavailable). Owner availability is not service lifecycle or health. Entries sort by project root, name, then ID. Text shows NAME, PROJECT, BRANCH, RUNTIME, OWNER, and a compact ID. An empty registry prints `No managed stacks found.` Use `--output-format json` to obtain diff --git a/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md index 4d51539261..7d26c5f7ac 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md @@ -35,7 +35,7 @@ caching, preparation, and cancellation; the CLI owns the temporary instance clea ## Flags and output `--stack` and `--stack-id` are mutually exclusive. `--runtime` defaults to `auto` for new stacks, -which selects the same runtime as `stack start` (Docker, then Podman, then native); +which selects the same runtime as `stack start` (Docker, then native); existing stacks reuse their persisted runtime without probing, and an explicit mismatch fails. With no `--capability`, every enabled creation from the effective project configuration is prepared. Repeated `--capability` includes each capability's configured companion services (Storage/Imgproxy, diff --git a/apps/cli/src/commands/experimental/stack/prepare/prepare.command.ts b/apps/cli/src/commands/experimental/stack/prepare/prepare.command.ts index 6ea91d399d..0d91f3e85b 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/prepare.command.ts +++ b/apps/cli/src/commands/experimental/stack/prepare/prepare.command.ts @@ -11,9 +11,9 @@ const config = { Flag.withDescription("Open an existing stack by id."), Flag.optional, ), - runtime: Flag.choice("runtime", ["auto", "docker", "podman", "native"] as const).pipe( + runtime: Flag.choice("runtime", ["auto", "docker", "native"] as const).pipe( Flag.withDescription( - "Runtime to use for a new stack. auto selects Docker, then Podman, then native, based on what is available.", + "Runtime to use for a new stack. auto selects Docker, then native, based on what is available.", ), Flag.withDefault("auto" as const), ), diff --git a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts index 32d6862aa3..cd4a7e1c46 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts @@ -49,18 +49,13 @@ const makeProject = (config: string) => interface FixtureOptions { readonly failPreparation?: boolean; readonly savedRuntime?: StackRuntime; - readonly engines?: { - readonly docker: ContainerEngineState; - readonly podman: ContainerEngineState; - }; + readonly engines?: { readonly docker: ContainerEngineState }; readonly runtimeInfo?: Layer.Layer; } const makeFixture = (root: string, options: FixtureOptions = {}) => { const { failPreparation = false } = options; - const engines = containerEngineSpawner( - options.engines ?? { docker: "missing", podman: "missing" }, - ); + const engines = containerEngineSpawner(options.engines ?? { docker: "missing" }); const createdRuntimes: Array = []; let openCount = 0; let prepareCount = 0; @@ -349,7 +344,7 @@ describe("stack prepare automatic runtime selection", () => { ); it.live("creates a Docker stack when the Docker engine answers", () => - selectRuntime({ engines: { docker: "running", podman: "running" } }).pipe( + selectRuntime({ engines: { docker: "running" } }).pipe( Effect.tap((fixture) => Effect.sync(() => { expect(fixture.createdRuntimes).toEqual(["docker"]); @@ -360,41 +355,22 @@ describe("stack prepare automatic runtime selection", () => { ), ); - it.live("creates a Podman stack when Docker is not installed and Podman answers", () => + it.live("creates a native stack on Linux when the Docker daemon is down", () => selectRuntime({ - engines: { docker: "missing", podman: "running" }, + engines: { docker: "stopped" }, runtimeInfo: mockRuntimeInfo({ platform: "linux", arch: "x64" }), }).pipe( Effect.tap((fixture) => Effect.sync(() => { - expect(fixture.createdRuntimes).toEqual(["podman"]); - expect(fixture.probes).toEqual(["docker", "podman"]); + expect(fixture.createdRuntimes).toEqual(["native"]); + expect(fixture.probes).toEqual(["docker"]); expect(fixture.runtimeNotices).toHaveLength(1); - expect(fixture.runtimeNotices[0]).toContain("uses the Podman runtime"); - expect(fixture.runtimeNotices[0]).toContain("supabase stack destroy"); + expect(fixture.runtimeNotices[0]).toContain("uses the native runtime"); }), ), ), ); - it.live( - "creates a native stack on Linux when the Docker daemon is down and Podman is absent", - () => - selectRuntime({ - engines: { docker: "stopped", podman: "missing" }, - runtimeInfo: mockRuntimeInfo({ platform: "linux", arch: "x64" }), - }).pipe( - Effect.tap((fixture) => - Effect.sync(() => { - expect(fixture.createdRuntimes).toEqual(["native"]); - expect(fixture.probes).toEqual(["docker", "podman"]); - expect(fixture.runtimeNotices).toHaveLength(1); - expect(fixture.runtimeNotices[0]).toContain("uses the native runtime"); - }), - ), - ), - ); - for (const [platform, arch] of [ ["win32", "x64"], ["darwin", "x64"], @@ -403,7 +379,7 @@ describe("stack prepare automatic runtime selection", () => { makeProject(databaseOnlyConfig).pipe( Effect.flatMap((root) => { const fixture = makeFixture(root, { - engines: { docker: "stopped", podman: "stopped" }, + engines: { docker: "stopped" }, runtimeInfo: mockRuntimeInfo({ platform, arch }), }); return stackPrepare(flags({ runtime: "auto" })).pipe( @@ -415,7 +391,7 @@ describe("stack prepare automatic runtime selection", () => { expect(error.reason).toBe("runtime"); expect(error.cause).toBeInstanceOf(StackRuntimeSelectionError); expect(error.message).toContain(`not supported on ${platform}/${arch}`); - expect(error.suggestion).toBe("Start Docker or Podman, then rerun the command."); + expect(error.suggestion).toBe("Start Docker, then rerun the command."); expect(fixture.createdRuntimes).toEqual([]); }), ), @@ -428,8 +404,8 @@ describe("stack prepare automatic runtime selection", () => { it.live("reuses a saved runtime without probing any engine", () => selectRuntime({ - savedRuntime: "podman", - engines: { docker: "running", podman: "running" }, + savedRuntime: "docker", + engines: { docker: "running" }, }).pipe( Effect.tap((fixture) => Effect.sync(() => { @@ -445,12 +421,12 @@ describe("stack prepare automatic runtime selection", () => { it.live("honors an explicit runtime without probing any engine", () => makeProject(databaseOnlyConfig).pipe( Effect.flatMap((root) => { - const fixture = makeFixture(root, { engines: { docker: "running", podman: "running" } }); - return stackPrepare(flags({ runtime: "podman" })).pipe( + const fixture = makeFixture(root, { engines: { docker: "stopped" } }); + return stackPrepare(flags({ runtime: "docker" })).pipe( Effect.provide(fixture.layer), Effect.tap(() => Effect.sync(() => { - expect(fixture.createdRuntimes).toEqual(["podman"]); + expect(fixture.createdRuntimes).toEqual(["docker"]); expect(fixture.probes).toEqual([]); expect(fixture.runtimeNotices).toEqual([]); }), diff --git a/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts index f4f6eef8c4..b488aad23d 100644 --- a/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts @@ -18,7 +18,7 @@ type TargetInput = { readonly projectRoot: string; readonly name?: string; readonly id?: string; - readonly runtime: "auto" | "docker" | "podman" | "native"; + readonly runtime: "auto" | "docker" | "native"; }; const workspace = Effect.gen(function* () { @@ -139,17 +139,6 @@ describe("stack target resolver", () => { }).pipe(Effect.provide(BunServices.layer)), ); - it.live("carries an explicit Podman runtime onto a new stack target", () => - Effect.gen(function* () { - const { root, resolve } = yield* workspace; - - const target = yield* resolve({ projectRoot: root, runtime: "podman" }); - - expect(target).toMatchObject({ runtime: "podman", hostRunning: false }); - expect(target.id).toBeUndefined(); - }).pipe(Effect.provide(BunServices.layer)), - ); - it.live("returns the canonical project root for a new stack reached through a symlink", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index 59b0bf2b38..e8e7ec3383 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -38,11 +38,11 @@ use `$SUPABASE_HOME/cache/stack`. Storage files use the caller-owned project dir `supabase/.temp/stack-uploads//`. Functions preparation may build the project's source. For a new stack, `--runtime auto` selects Docker when `docker version` reaches its daemon, then -Podman when `podman info` reaches its engine, then native on Linux x64/arm64 and macOS arm64. Each -probe is bounded by 10 seconds. Without a reachable engine on other platforms, the command fails and -asks the user to start Docker or Podman. When auto selection skips Docker, an info line names the -saved Podman or native runtime and how to switch to Docker. An existing stack keeps its saved -runtime and runs no probe. Explicit `--runtime docker`, `podman`, or `native` has no fallback. +native on Linux x64/arm64 and macOS arm64. The probe is bounded by 10 seconds. Without a reachable +engine on other platforms, the command fails and asks the user to start Docker. When auto selection +skips Docker, an info line names the saved native runtime and how to switch to Docker. An existing +stack keeps its saved runtime and runs no probe. Explicit `--runtime docker` or `native` has no +fallback. When an explicit or saved Docker runtime is unreachable, the reported failure suggests starting Docker, and `--runtime native` for a new stack on platforms that support native. Explicit `--runtime native` on a platform with no native artifacts fails before creating a stack. diff --git a/apps/cli/src/commands/experimental/stack/start/start.command.ts b/apps/cli/src/commands/experimental/stack/start/start.command.ts index 4f9df6a258..e5a84466b8 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.command.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.command.ts @@ -20,9 +20,9 @@ const config = { Flag.withDescription("Open an existing stack by id."), Flag.optional, ), - runtime: Flag.choice("runtime", ["auto", "docker", "podman", "native"] as const).pipe( + runtime: Flag.choice("runtime", ["auto", "docker", "native"] as const).pipe( Flag.withDescription( - "Runtime to use for a new stack. auto selects Docker, then Podman, then native, based on what is available.", + "Runtime to use for a new stack. auto selects Docker, then native, based on what is available.", ), Flag.withDefault("auto" as const), ), @@ -44,7 +44,7 @@ export const stackStartCommand = Command.make("start", config).pipe( Command.withDescription( "Create or resume a managed local Supabase stack using supabase/config.toml when present. " + "Without a config file, default settings are used and no file is created. " + - "For a new stack, auto selects Docker when its engine is reachable, then Podman, then native on supported platforms; the choice is persisted for that stack. " + + "For a new stack, auto selects Docker when its engine is reachable, then native on supported platforms; the choice is persisted for that stack. " + "Explicit runtime choices are honored. Values support explicit env(NAME) references and automatic SUPABASE_* overrides.", ), Command.withShortDescription("Start a managed local stack"), diff --git a/apps/cli/src/commands/experimental/stack/start/start.handler.ts b/apps/cli/src/commands/experimental/stack/start/start.handler.ts index dd9324f72f..b8079078f8 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.handler.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.handler.ts @@ -14,6 +14,7 @@ import { statusEnvPointer } from "./start-summary.format.ts"; import { automaticRuntimeNotice, selectStackRuntime, + type StackRuntime, } from "../../../../command-internal/stack-runtime.ts"; import { RuntimeInfo } from "../../../../shared/runtime/runtime-info.service.ts"; import { Effect, FileSystem, Fiber, Option, Path, Redacted, Ref } from "effect"; @@ -121,7 +122,7 @@ const dockerUnavailableSuggestion = ( const stackAcquireError = ( cause: StackError, runtimeContext: { - readonly selectedRuntime: "native" | "docker" | "podman"; + readonly selectedRuntime: StackRuntime; readonly runtime: { readonly platform: string; readonly arch: string }; readonly creating: boolean; }, diff --git a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts index c1267d0a4f..8c794ae654 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts @@ -34,7 +34,6 @@ import { withEnvVar, } from "../../../../../tests/helpers/command-mocks.ts"; import { mockOutput, mockRuntimeInfo, mockTty } from "../../../../../tests/helpers/mocks.ts"; -import { containerEngineSpawner } from "../../../../../tests/helpers/child-process-spawner.ts"; import { DbConnection, type DbSession, @@ -1208,48 +1207,6 @@ describe("experimental stack start", () => { }).pipe(Effect.provide(BunServices.layer)), ); - it.live("reports the saved runtime when automatic selection creates a Podman stack", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-auto-podman-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "auto-podman"\n[edge_runtime]\nenabled = false\n', - ); - const output = mockOutput(); - const engines = containerEngineSpawner({ docker: "stopped", podman: "running" }); - const target = Layer.succeed(StackTargetResolver, { - resolve: () => Effect.succeed({ projectRoot: root, hostRunning: false }), - }); - yield* stackStart({ - ...flags([ - "rest", - "auth", - "realtime", - "storage", - "functions", - "studio", - "mail", - "analytics", - "pooler", - ]), - runtime: "auto", - }).pipe( - Effect.provide( - Layer.mergeAll(layers(root, fakeStack(), output, false), target, engines.layer), - ), - ); - expect(engines.spawned.map(({ command }) => command)).toEqual(["docker", "podman"]); - expect(output.messages).toContainEqual({ - type: "info", - message: expect.stringContaining( - "Docker didn't answer, so this new stack uses the Podman runtime", - ), - }); - }).pipe(Effect.provide(BunServices.layer)), - ); - // The `#!/bin/sh` shim is never picked up on Windows, which only resolves `docker.exe` on PATH. it.live.skipIf(process.platform === "win32")( "leaves no stack registered when a new stack's owner cannot reach the Docker daemon", @@ -1333,7 +1290,7 @@ describe("experimental stack start", () => { Effect.flip, Effect.provide(Layer.mergeAll(layers(root, fakeStack(), output, false), target, api)), ); - expect(error.message).toContain("docker ps"); + expect(error.message).toContain("docker context show"); expect(error).toBeInstanceOf(StackCommandStartError); if (error instanceof StackCommandStartError) { expect(error.reason).toBe("runtime"); diff --git a/apps/cli/src/commands/experimental/stack/status/status.handler.ts b/apps/cli/src/commands/experimental/stack/status/status.handler.ts index c0450d6659..8ef1b44d5d 100644 --- a/apps/cli/src/commands/experimental/stack/status/status.handler.ts +++ b/apps/cli/src/commands/experimental/stack/status/status.handler.ts @@ -57,7 +57,7 @@ type StackReport = { readonly project_root: string; readonly branch_context: string; }; - readonly runtime: "native" | "docker" | "podman"; + readonly runtime: "native" | "docker"; readonly owner: "reachable" | "unavailable"; readonly lifecycle: Observation["lifecycle"] | null; readonly readiness: "unavailable" | "starting" | "sleeping" | "stopped" | "ready" | "unhealthy"; @@ -156,7 +156,7 @@ const reportFor = ( readonly branchContext: string; readonly stackName: string; }; - readonly runtime: "native" | "docker" | "podman"; + readonly runtime: "native" | "docker"; }, owner: StackReport["owner"], observed: ReadonlyArray, diff --git a/apps/cli/tests/helpers/child-process-spawner.ts b/apps/cli/tests/helpers/child-process-spawner.ts index 93200bae25..241cfcccd2 100644 --- a/apps/cli/tests/helpers/child-process-spawner.ts +++ b/apps/cli/tests/helpers/child-process-spawner.ts @@ -106,13 +106,10 @@ export function mockChildProcessSpawner( export type ContainerEngineState = "missing" | "stopped" | "running"; /** - * Spawner for a host whose `docker` and `podman` commands behave as `engines` describe. Other - * commands fail with `NotFound`, or run on the real spawner through `hidingLayer`. + * Spawner for a host whose `docker` command behaves as `engines` describes. Other commands fail + * with `NotFound`, or run on the real spawner through `hidingLayer`. */ -export function containerEngineSpawner(engines: { - readonly docker: ContainerEngineState; - readonly podman: ContainerEngineState; -}) { +export function containerEngineSpawner(engines: { readonly docker: ContainerEngineState }) { const spawned: SpawnRecord[] = []; const notFound = (command: string) => PlatformError.systemError({ @@ -125,7 +122,7 @@ export function containerEngineSpawner(engines: { ChildProcessSpawner.make((command) => { const cmd = Predicate.isTagged(command, "StandardCommand") ? command.command : ""; const args = Predicate.isTagged(command, "StandardCommand") ? command.args : []; - if (cmd !== "docker" && cmd !== "podman") + if (cmd !== "docker") return delegate === undefined ? Effect.fail(notFound(cmd)) : delegate.spawn(command); spawned.push({ command: cmd, args }); const state = engines[cmd]; diff --git a/packages/stack/README.md b/packages/stack/README.md index 5363725a53..e7f266be73 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -14,7 +14,7 @@ import { postgres } from "@supabase/stack/commands"; const stack = await create({ projectRoot: process.cwd(), - runtime: "native", // or "docker" / "podman" + runtime: "native", // or "docker" stateRoot: "/tmp/example-stack/state", cacheRoot: "/tmp/example-stack/cache", }); @@ -59,11 +59,11 @@ await stack.close(); // disconnects this client Creating a service records its definition. Configured public ports are bound during startup and retained across normal stop/start and owner reopening. An occupied saved port reports a conflict instead of moving. Automatic ports avoid numbers saved by any stack under the same `stateRoot`; a fixed port is decided by binding it, so another stack's saved port blocks only while something listens on it; that conflict names the stack that saved the port. Omitted public endpoints are not exposed. -Native public listeners bind to loopback. Docker and Podman public proxies bind all interfaces so services inside the container network can reach them; those listeners are reachable from the LAN according to the host firewall. +Native public listeners bind to loopback. Docker public proxies bind all interfaces so services inside the container network can reach them; those listeners are reachable from the LAN according to the host firewall. Functions use a package-provided, self-contained Edge Runtime main service unless the configuration supplies `bootstrap` source; only an explicit `bootstrap` is saved with the service definition. Database versions belong in `config.version`; other recipes accept an optional top-level artifact `version`. -On Linux, native Functions project files must be outside `/tmp`: Edge Runtime uses a private filesystem at that path. Docker and Podman mount project files at a separate runtime path. +On Linux, native Functions project files must be outside `/tmp`: Edge Runtime uses a private filesystem at that path. Docker mounts project files at a separate runtime path. `open({ id, stateRoot, cacheRoot })` reconnects to a saved stack. The package stores the stack document at `//state.json` and service data at `//data/`. `discover({ stateRoot })` lists saved definitions and port assignments separately from live-owner availability. It skips each entry that cannot be read or decoded and reports it to `onInvalidState(id, error)`; only a failure to read `stateRoot` itself fails discovery. Port allocation skips the same entries. Offline definitions are not live lifecycle observations. @@ -77,7 +77,7 @@ Pass `startOwner: true` to `open` when live status and other owner-backed operat `create({ ..., startOwner: true })` starts a detached stack's owner immediately and lets it register the stack under its lease, as a session stack always does. If the owner fails to start or the launch is interrupted before it reports ready, the owner removes that registration and `create` fails with the launch error, so a failed launch leaves no stack behind. -`destroy` normally returns `{ runtimeCleanup: "complete" }`. When no owner is running and a new owner cannot start because the stack's container engine reports that its daemon cannot be reached, `destroy` takes the stack's lease, so no owner can start meanwhile, and removes the local registration, port claims and host data anyway and returns `{ runtimeCleanup: "skipped", engine, cleanupCommands }`; its containers and any database data in engine volumes remain, and `cleanupCommands` are the shell commands that remove them once the engine is running. If some host data cannot be deleted by the current user, `destroy` fails before removing anything so it can be retried with the engine running. +`destroy` normally returns `{ runtimeCleanup: "complete" }`. When no owner is running and a new owner cannot start because the stack's container engine reports that its daemon cannot be reached, `destroy` confirms that no owner holds the stack's lease and leaves the registration, port claims and host data untouched, returning `{ runtimeCleanup: "skipped", engine }`. When an owner does run but some claims (for example a container on a different daemon) cannot be reconciled, `destroy` likewise keeps the stack registered and fails, listing the remaining claims and the `claims.json` path to edit if the original engine is permanently gone. Either way, retry `destroy` once the engine is reachable; there is no `cleanupCommands` field. The stack owns database, Functions bootstrap, and command-job directories below its data directory. Storage uploads remain at the caller-supplied Storage `filePath` and are preserved when the stack is destroyed; the caller owns that directory. Host metadata remains under `stateRoot`; native database data uses host files. Docker database data normally uses a managed volume, while existing host data is retained through the host-backed fallback. A host marker records the selected Docker storage and detects a missing or mismatched volume; deleting that volume loses the associated database data. Native snapshot entries live below `cacheRoot`. Docker snapshots share the managed data volume in a separate namespace derived from `cacheRoot`, so they survive source destruction and can use filesystem cloning. A Docker cache hit requires the same daemon, `stateRoot`, and `cacheRoot`. There is no portable tar snapshot API. diff --git a/packages/stack/src/Commands.integration.test.ts b/packages/stack/src/Commands.integration.test.ts index 21d81d129a..62eadbcdef 100644 --- a/packages/stack/src/Commands.integration.test.ts +++ b/packages/stack/src/Commands.integration.test.ts @@ -25,6 +25,8 @@ import { makeDatabase } from "./services/Database.ts"; import { makeService } from "./Service.ts"; import { bindTcp, serveTcp } from "./Proxy.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; +import { noContainerClaims, noDirectoryClaims } from "../tests/claims.ts"; +import { dockerEngineTarget } from "../tests/engine-target.ts"; const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; @@ -36,11 +38,15 @@ const makeTestCommandRunner = (options: { readonly stackId: string; readonly root: string; readonly cacheRoot: string; - readonly runtime: "native" | "docker" | "podman"; + readonly runtime: "native" | "docker"; }) => - Layer.build(CommandRunner.layer(options)).pipe( - Effect.map((context) => Context.get(context, CommandRunner.Service)), - ); + Layer.build( + CommandRunner.layer({ + ...options, + claims: noContainerClaims, + ...(options.runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), + }), + ).pipe(Effect.map((context) => Context.get(context, CommandRunner.Service))); const runPostgres = ( runner: CommandRunner.Interface, @@ -84,11 +90,14 @@ describe("finite PostgreSQL commands", { timeout: 180_000 }, () => { ? yield* makeDockerDatabaseRoot("stack-tools-", stackId) : yield* fs.makeTempDirectoryScoped({ prefix: "stack-tools-" }); const database = yield* makeDatabase({ + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, root, cacheRoot, stackId, instanceId: "database", runtime, + ...(runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), }); const service = yield* makeService(database.definition, { id: "database", @@ -234,11 +243,14 @@ describe("finite PostgreSQL commands", { timeout: 180_000 }, () => { ? yield* makeDockerDatabaseRoot(`stack-pgprove-${major}-`, stackId) : yield* fs.makeTempDirectoryScoped({ prefix: `stack-pgprove-${major}-` }); const database = yield* makeDatabase({ + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, root, cacheRoot, stackId, instanceId: "database", runtime, + ...(runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), }); const service = yield* makeService(database.definition, { id: "database", @@ -270,8 +282,12 @@ describe("finite PostgreSQL commands", { timeout: 180_000 }, () => { PGPASSWORD: `pgprove-${runtime}-${major}-password`, PGDATABASE: "postgres", }; - const tests = `${root}/tests`; - yield* fs.makeDirectory(tests, { recursive: true }); + // Ownership is by location: pg_prove's mount source is a caller path and must + // live outside the stack's data root, not merely outside the database's own + // instance root. + const tests = yield* fs.makeTempDirectoryScoped({ + prefix: `stack-pgprove-${major}-tests-`, + }); yield* fs.writeFileString(`${tests}/main.sql`, "\\ir included.sql\n"); yield* fs.writeFileString(`${tests}/included.sql`, "\\i nested.sql\n"); yield* fs.writeFileString( diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index 4315b18279..3d45e023d9 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -5,7 +5,7 @@ import { makePorts, PortError } from "./Ports.ts"; import { bindTcp, serveTcp, type BackendAddress, type ProxyError } from "./Proxy.ts"; import { makeHttpProxy, type HttpProxy, type HttpRoute } from "./HttpProxy.ts"; -export type NetworkRuntime = "native" | "docker" | "podman"; +export type NetworkRuntime = "native" | "docker"; type RouteContribution = Pick< HttpRoute, @@ -91,12 +91,7 @@ const makeNetwork = (options: { const listenHost = options.runtime === "native" ? "127.0.0.1" : "0.0.0.0"; const hostAddress = "127.0.0.1"; - const runtimeAddress = - options.runtime === "native" - ? "127.0.0.1" - : options.runtime === "docker" - ? DOCKER_HOST_ALIAS - : "host.containers.internal"; + const runtimeAddress = options.runtime === "native" ? "127.0.0.1" : DOCKER_HOST_ALIAS; const routeKey = (route: Pick) => `${route.id}:${route.prefix}`; diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index 9271d7ea09..cefe2c8142 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -15,6 +15,7 @@ import { } from "effect"; import { HttpClient } from "effect/unstable/http"; import { ChildProcessSpawner } from "effect/unstable/process"; +import type { PlatformError } from "effect/PlatformError"; import type { Rpc, RpcGroup } from "effect/unstable/rpc"; import { failureMessage } from "./internal/failure-message.ts"; import * as Network from "./Network.ts"; @@ -62,6 +63,11 @@ import { } from "./services/Catalog.ts"; import type { CatalogError } from "./services/Recipe.ts"; import * as Container from "./runtime/Container.ts"; +import * as Claims from "./namespace/Claims.ts"; +import { namespaceError } from "./namespace/Capabilities.ts"; +import { lstatPath } from "./namespace/drivers/FileSystem.ts"; +import * as Paths from "./namespace/Paths.ts"; +import { resolveNativeRuntimeRootForRecovery } from "./runtime/postgres-user.ts"; import { projectSegmentFor } from "./identity/Identity.ts"; import { stackError, type OwnerRpc } from "./Rpc.ts"; import * as StackNamespace from "./StackNamespace.ts"; @@ -75,6 +81,8 @@ export interface OwnerOptions { readonly cacheRoot: string; /** Shares one host-gateway probe with the host's other container runtimes. */ readonly hostGateway?: Container.HostGateway; + /** The engine endpoint and identity resolved once at startup; absent for a native stack. */ + readonly engineTarget?: Container.EngineTarget; } type OwnerRpcs = RpcGroup.Rpcs; @@ -86,13 +94,67 @@ type Handlers = { ) => Rpc.ResultFrom; }; -/** Removes containers labeled with this stack and data root; native stacks own none. */ +/** True when `target` exists in any form (a real entry or a symlink); lstat-style, never follows. */ +const directoryClaimExists = Effect.fn("Owner.directoryClaimExists")(function* (target: string) { + return (yield* lstatPath(target)) !== undefined; +}); + +/** + * Removes this stack's claimed resources by exact identity, through the owner's pinned engine target. + * Container claims from another or unknown daemon, and directory claims outside every owned root, + * are kept and reported, so recovery (possibly as root) never deletes what it does not own. A claim + * naming a path that no longer exists in any form is simply dropped, nothing left to remove. + */ export const sweepContainers = Effect.fn("Owner.sweepContainers")(function* ( + state: StackNamespace.Interface, saved: Pick, root: string, + target: Container.EngineTarget | undefined, ) { - if (saved.runtime !== "native") - yield* Container.removeStackContainers({ engine: saved.runtime, stackId: saved.id, root }); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + // Missing, foreign, or symlinked is "no such owned root", not a sweep failure: anything claimed + // under it is reported, not deleted through whatever replaced it. + const nativeRuntimeRoot = yield* resolveNativeRuntimeRootForRecovery().pipe(Effect.option); + const ownedRoots = [root, ...Option.toArray(nativeRuntimeRoot)]; + yield* Claims.reconcile( + state, + saved.id, + ( + claim, + ): Effect.Effect< + "removed" | "kept", + Container.ContainerError | PlatformError | StackNamespace.NamespaceError, + ChildProcessSpawner.ChildProcessSpawner + > => { + if (claim.kind === "directory") + return directoryClaimExists(claim.id).pipe( + Effect.flatMap((present) => + !present + ? Effect.succeed("removed" as const) + : Paths.isWithinOwnedRoots(fs, path, claim.id, ownedRoots, (_operation, cause) => + namespaceError("cleanup", cause), + ).pipe( + Effect.flatMap((within) => + within + ? fs + .remove(claim.id, { recursive: true, force: true }) + .pipe(Effect.as("removed" as const)) + : Effect.logWarning( + `Claimed directory ${claim.id} resolves outside every owned root; keeping it`, + ).pipe(Effect.as("kept" as const)), + ), + ), + ), + ); + if (target === undefined) return Effect.succeed("kept" as const); + if (claim.daemonId === undefined || claim.daemonId !== target.daemonId) + return Effect.succeed("kept" as const); + return Container.removeContainerById({ target, id: claim.id }).pipe( + Effect.as("removed" as const), + ); + }, + ); }); type NamespaceError = @@ -291,6 +353,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { return yield* withCredentials(creation, credentials); }); + const claims = Claims.forStack(options.state, stackId); const recipeFor = (creation: ServiceCreation, id: string) => makeServiceRecipe(creation, { stackId, @@ -300,7 +363,10 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { cacheRoot: options.cacheRoot, runtime, helpers, + containerClaims: claims.containers, + directoryClaims: claims.directories, ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), + ...(options.engineTarget === undefined ? {} : { engineTarget: options.engineTarget }), }).pipe(Effect.provideContext(services)); const persistCreation = (entry: Pick, creation: ServiceCreation) => @@ -675,7 +741,12 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ), }; - const sweep = sweepContainers(options.saved, options.root).pipe(Effect.provideContext(services)); + const sweep = sweepContainers( + options.state, + options.saved, + options.root, + options.engineTarget, + ).pipe(Effect.provideContext(services)); const getStackCredentials = readSaved.pipe( Effect.flatMap(({ credentials }) => credentials === undefined @@ -699,7 +770,34 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { destroy: orchestrator.destroyNamespace.pipe( Effect.andThen(network.release), Effect.andThen(sweep), - Effect.andThen(options.state.remove(stackId)), + // A claim reconcile deliberately kept (for example one recorded against a different + // daemon) must not be lost to a full deregistration; the stack stays registered so a + // later acquisition's reconcile, against the right daemon, can still finish it. + Effect.andThen(options.state.readClaims(stackId)), + Effect.flatMap((remaining) => + remaining.length === 0 + ? options.state.remove(stackId) + : Effect.gen(function* () { + const currentDaemonId = options.engineTarget?.daemonId; + const claimsPath = path.join(options.state.root, stackId, Claims.CLAIMS_FILE); + const listed = remaining + .map( + (claim) => + ` - ${claim.kind} ${claim.id} (daemon ${claim.daemonId ?? "unknown"})`, + ) + .join("\n"); + return yield* new StackNamespace.NamespaceError({ + operation: "destroy", + message: [ + `${remaining.length} resource claim(s) could not be reconciled; stack stays registered for retry.`, + listed, + `Current daemon: ${currentDaemonId ?? "unreachable"}.`, + "To recover: stop this owner, confirm the lease is free, then run `supabase stack destroy` again once the original engine is back. " + + `If that engine is permanently gone, remove the listed claim entries from ${claimsPath} and retry.`, + ].join("\n"), + }); + }), + ), definitionGate.withPermits(1), Effect.withSpan("Owner.destroyNamespace"), ), diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index a89db6e76e..2ff53043d4 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -10,9 +10,10 @@ import { ownerExitProbe, waitForOwnerExit, } from "./HostProcess.ts"; -import { makeContainerRuntime } from "./runtime/Container.ts"; +import { makeContainerRuntime, resolveEngineTarget } from "./runtime/Container.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; import { shutdownOwner } from "../tests/owner.ts"; +import { noContainerClaims } from "../tests/claims.ts"; const helperImage = "public.ecr.aws/docker/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; @@ -66,18 +67,6 @@ const removeContainers = (stackId: string, dataRoot: string) => ), ); -const createOwnedContainer = (name: string, stackId: string, dataRoot: string) => - docker([ - "create", - "--name", - name, - "--label", - `com.supabase.stack=${stackId}`, - "--label", - `com.supabase.stack-root=${dataRoot}`, - helperImage, - ]); - const startHost = (stateRoot: string, cacheRoot: string, stackId: string, projectRoot: string) => Effect.gen(function* () { const state = yield* stateFor(stateRoot); @@ -114,7 +103,13 @@ it.live.skipIf(process.platform === "win32")( const rootA = path.dirname(path.dirname(dataA)); const rootB = path.dirname(path.dirname(dataB)); const cacheRoot = `${base}/cache`; - const helper = yield* makeContainerRuntime({ engine: "docker", root: dataA }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const target = yield* resolveEngineTarget(spawner); + const helper = yield* makeContainerRuntime({ + claims: noContainerClaims, + target, + root: dataA, + }); yield* helper.prepare(helperImage); let activeA: { readonly pid: number; readonly port: number } | undefined; let activeB: { readonly pid: number; readonly port: number } | undefined; @@ -141,41 +136,16 @@ it.live.skipIf(process.platform === "win32")( yield* removeContainers(stackId, dataB).pipe(Effect.ignore); }), ); - const staleAtStartup = yield* createOwnedContainer( - `stack-stale-${stackId}-startup`, - stackId, - dataA, - ); const accessA = yield* startHost(rootA, cacheRoot, stackId, `${base}/project-a`); const endpointA = accessA.endpoint; activeA = endpointA; stoppedA = false; - expect(staleAtStartup.length).toBeGreaterThan(0); - expect( - yield* containers(stackId, dataA), - "startup sweep removes A stale container", - ).toEqual([]); - const staleForA = yield* createOwnedContainer( - `stack-stale-${stackId}-parallel`, - stackId, - dataA, - ); - const staleForB = yield* createOwnedContainer( - `stack-stale-${stackId}-other-root`, - stackId, - dataB, - ); const accessB = yield* startHost(rootB, cacheRoot, stackId, `${base}/project-b`); const endpointB = accessB.endpoint; activeB = endpointB; stoppedB = false; - expect(staleForA.length).toBeGreaterThan(0); - expect(staleForB.length).toBeGreaterThan(0); - expect(yield* containers(stackId, dataA)).toEqual([staleForA]); - expect( - yield* containers(stackId, dataB), - "startup sweep removes B stale container", - ).toEqual([]); + expect(yield* containers(stackId, dataA)).toEqual([]); + expect(yield* containers(stackId, dataB)).toEqual([]); const clientA = yield* ownerClient(accessA); const clientB = yield* ownerClient(accessB); diff --git a/packages/stack/src/StackHost.container-sweep-retry.integration.test.ts b/packages/stack/src/StackHost.container-sweep-retry.integration.test.ts index e652e857ed..1a52a7bf73 100644 --- a/packages/stack/src/StackHost.container-sweep-retry.integration.test.ts +++ b/packages/stack/src/StackHost.container-sweep-retry.integration.test.ts @@ -1,61 +1,310 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Context, Deferred, Effect, FileSystem, Layer, Sink, Stream } from "effect"; +import { Context, Effect, FileSystem, Layer, Path, Sink, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import * as StackNamespace from "./StackNamespace.ts"; import * as Owner from "./Owner.ts"; -import * as CommandRunner from "./host/CommandRunner.ts"; -import { bindControl, makeRuntime } from "./StackHost.ts"; -import { shutdownOwner } from "../tests/owner.ts"; +import type { EngineTarget } from "./runtime/Container.ts"; +import { NativeRuntimeRootBase, nativeRuntimeRootPath } from "./runtime/postgres-user.ts"; const stateFor = (root: string) => Layer.build(StackNamespace.layer({ root })).pipe( Effect.map((context) => Context.get(context, StackNamespace.Service)), ); -it.live("retains saved state when destroy sweep fails and retries after engine recovery", () => +const containerId = "claimed0000000000000000000000000001"; + +/** Resolved once, like an owner's own startup would, and pinned for the whole test. */ +const engineTarget: EngineTarget = { engine: "docker", argv: [], daemonId: "current-daemon" }; + +it.live( + "retains a claim when the engine is unreachable during destroy, and finishes it once the engine recovers", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-container-sweep-retry-" }); + const state = yield* stateFor(`${root}/state`); + const saved = { + id: "stack", + runtime: "docker" as const, + identity: { projectRoot: root, branchContext: "main", stackName: "sweep-retry" }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + // A now-dead owner already claimed this container before being interrupted. Recorded + // against the daemon this reconcile run will later match, once it recovers. + yield* state.claim(saved.id, { + kind: "container", + id: containerId, + daemonId: "current-daemon", + }); + let engineReachable = false; + let rmCalls = 0; + const engine = ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command) || command.command !== "docker") + return Effect.die("Unexpected child process command"); + const isClaimRemoval = command.args[0] === "rm" && command.args.includes(containerId); + if (isClaimRemoval) rmCalls++; + const fail = isClaimRemoval && !engineReachable; + return Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(0), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(fail ? 1 : 0)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + stdout: Stream.empty, + stderr: fail + ? Stream.succeed( + new TextEncoder().encode( + "Cannot connect to the Docker daemon at tcp://127.0.0.1:1. Is the docker daemon running?", + ), + ) + : Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }), + ); + }); + const ownerLayer = Owner.layer({ + saved, + root: `${root}/data`, + cacheRoot: `${root}/cache`, + engineTarget, + }).pipe( + Layer.provide( + Layer.merge( + Layer.succeed(StackNamespace.Service, state), + Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, engine), + ), + ), + ); + const firstOwner = yield* Layer.build(ownerLayer).pipe( + Effect.map((context) => Context.get(context, Owner.Service)), + ); + + const firstAttempt = yield* firstOwner.namespace.destroy.pipe(Effect.exit); + expect(firstAttempt._tag).toBe("Failure"); + expect(rmCalls).toBe(1); + expect(yield* state.read(saved.id)).toBeDefined(); + expect((yield* state.readClaims(saved.id)).map((claim) => claim.id)).toEqual([containerId]); + + // The engine is reachable again: a later acquisition's reconcile finishes the cleanup. + engineReachable = true; + const retryOwner = yield* Layer.build(ownerLayer).pipe( + Effect.map((context) => Context.get(context, Owner.Service)), + ); + yield* retryOwner.namespace.destroy; + expect(rmCalls).toBe(2); + expect(yield* state.read(saved.id)).toBeUndefined(); + expect(yield* state.readClaims(saved.id)).toEqual([]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live( + "removes a claimed directory left behind under /tmp and unclaims it, for a native stack with no engine", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-directory-sweep-" }); + const state = yield* stateFor(`${root}/state`); + const saved = { + id: "stack", + runtime: "native" as const, + identity: { projectRoot: root, branchContext: "main", stackName: "directory-sweep" }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + // A now-dead owner claimed this short-socket directory, nested under the native runtime + // root, before being killed; not scoped, since the reconcile under test is itself the one + // that removes it. + const runtimeRootBase = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-directory-sweep-runtime-root-", + }); + const runtimeRoot = nativeRuntimeRootPath(path, runtimeRootBase, process.getuid?.() ?? 0); + const socketDirectory = path.join(runtimeRoot, "pg-leftover"); + yield* fs.makeDirectory(socketDirectory, { recursive: true, mode: 0o700 }); + yield* fs.writeFileString(`${socketDirectory}/pg_hba.conf`, "local all all trust\n"); + yield* state.claim(saved.id, { kind: "directory", id: socketDirectory }); + const ownerLayer = Owner.layer({ + saved, + root: `${root}/data`, + cacheRoot: `${root}/cache`, + }).pipe( + Layer.provide( + Layer.merge( + Layer.succeed(StackNamespace.Service, state), + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), + ), + ), + ), + Layer.provide(Layer.succeed(NativeRuntimeRootBase, runtimeRootBase)), + ); + const owner = yield* Layer.build(ownerLayer).pipe( + Effect.map((context) => Context.get(context, Owner.Service)), + ); + yield* owner.namespace.destroy; + expect(yield* fs.exists(socketDirectory)).toBe(false); + expect(yield* state.readClaims(saved.id)).toEqual([]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live( + "keeps and reports a claimed directory that resolves outside every owned root, for a native stack with no engine", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-directory-sweep-outside-", + }); + const state = yield* stateFor(`${root}/state`); + const saved = { + id: "stack", + runtime: "native" as const, + identity: { + projectRoot: root, + branchContext: "main", + stackName: "directory-sweep-outside", + }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + // A foreign directory under neither the data root nor the native runtime root: never ours + // by location, however it got its name, so recovery must leave it alone. + const foreign = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-directory-sweep-foreign-", + }); + yield* fs.writeFileString(`${foreign}/marker`, "not ours\n"); + yield* state.claim(saved.id, { kind: "directory", id: foreign }); + const ownerLayer = Owner.layer({ + saved, + root: `${root}/data`, + cacheRoot: `${root}/cache`, + }).pipe( + Layer.provide( + Layer.merge( + Layer.succeed(StackNamespace.Service, state), + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), + ), + ), + ), + ); + const owner = yield* Layer.build(ownerLayer).pipe( + Effect.map((context) => Context.get(context, Owner.Service)), + ); + const failure = yield* owner.namespace.destroy.pipe(Effect.flip); + expect(failure.message).toContain(`directory ${foreign} (daemon unknown)`); + expect(yield* fs.exists(foreign)).toBe(true); + expect(yield* state.read(saved.id)).toBeDefined(); + expect((yield* state.readClaims(saved.id)).map((claim) => claim.id)).toEqual([foreign]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live( + "keeps and reports a claim nested under a runtime root a foreign uid could have taken over", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-directory-sweep-takeover-", + }); + const state = yield* stateFor(`${root}/state`); + const saved = { + id: "stack", + runtime: "native" as const, + identity: { + projectRoot: root, + branchContext: "main", + stackName: "directory-sweep-takeover", + }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + // Tests can't create another uid's files, so a group/world-writable leaf stands in for one + // recreated by a foreign uid after the original was removed. + const runtimeRootBase = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-directory-sweep-takeover-base-", + }); + const runtimeRoot = nativeRuntimeRootPath(path, runtimeRootBase, process.getuid?.() ?? 0); + const claimed = path.join(runtimeRoot, "pg-claimed"); + yield* fs.makeDirectory(claimed, { recursive: true }); + yield* fs.chmod(runtimeRoot, 0o777); + yield* fs.writeFileString(`${claimed}/marker`, "not ours\n"); + yield* state.claim(saved.id, { kind: "directory", id: claimed }); + const ownerLayer = Owner.layer({ + saved, + root: `${root}/data`, + cacheRoot: `${root}/cache`, + }).pipe( + Layer.provide( + Layer.merge( + Layer.succeed(StackNamespace.Service, state), + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), + ), + ), + ), + Layer.provide(Layer.succeed(NativeRuntimeRootBase, runtimeRootBase)), + ); + const owner = yield* Layer.build(ownerLayer).pipe( + Effect.map((context) => Context.get(context, Owner.Service)), + ); + const failure = yield* owner.namespace.destroy.pipe(Effect.flip); + expect(failure.message).toContain(`directory ${claimed} (daemon unknown)`); + expect(yield* fs.exists(claimed)).toBe(true); + expect((yield* state.readClaims(saved.id)).map((claim) => claim.id)).toEqual([claimed]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live("drops a claimed directory that no longer exists, for a native stack with no engine", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-container-sweep-retry-" }); + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-directory-sweep-gone-" }); const state = yield* stateFor(`${root}/state`); const saved = { id: "stack", - runtime: "docker" as const, - identity: { projectRoot: root, branchContext: "main", stackName: "sweep-retry" }, + runtime: "native" as const, + identity: { projectRoot: root, branchContext: "main", stackName: "directory-sweep-gone" }, instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, ports: [], }; yield* state.save(saved); - let listCalls = 0; - const engine = ChildProcessSpawner.make((command) => { - if (!ChildProcess.isStandardCommand(command) || command.command !== "docker") - return Effect.die("Unexpected child process command"); - const isStackSweep = - command.args[0] === "ps" && - command.args.includes(`label=com.supabase.stack=${saved.id}`) && - command.args.includes(`label=com.supabase.stack-root=${root}/data`); - const failList = isStackSweep && ++listCalls === 1; - return Effect.succeed( - ChildProcessSpawner.makeHandle({ - pid: ChildProcessSpawner.ProcessId(0), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(failList ? 1 : 0)), - isRunning: Effect.succeed(false), - kill: () => Effect.void, - stdin: Sink.drain, - stdout: Stream.empty, - stderr: failList - ? Stream.succeed(new TextEncoder().encode("temporary engine failure")) - : Stream.empty, - all: Stream.empty, - getInputFd: () => Sink.drain, - getOutputFd: () => Stream.empty, - unref: Effect.succeed(Effect.void), - }), - ); - }); + // Never created on disk: the owner that journaled it crashed before the directory existed. + const gone = path.join(root, "pg-never-existed"); + yield* state.claim(saved.id, { kind: "directory", id: gone }); const ownerLayer = Owner.layer({ saved, root: `${root}/data`, @@ -64,54 +313,165 @@ it.live("retains saved state when destroy sweep fails and retries after engine r Layer.provide( Layer.merge( Layer.succeed(StackNamespace.Service, state), - Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, engine), + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), + ), ), ), ); - const baseOwner = yield* Layer.build(ownerLayer).pipe( - Effect.map((context) => Context.get(context, Owner.Service)), - ); - const owner = baseOwner; - const acquired = yield* bindControl(); - const runnerLayer = yield* Layer.build( - CommandRunner.layer({ - stackId: saved.id, - root: `${root}/data`, - cacheRoot: `${root}/cache`, - runtime: "docker", - }).pipe(Layer.provide(Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, engine))), - ); - const runtime = yield* makeRuntime( - owner, - { - endpoint: { - stackId: saved.id, - identity: saved.identity, - pid: process.pid, - port: acquired.port, - release: "test", - }, - secret: "test-secret", - }, - acquired.server, - acquired.closeConnections, - ).pipe( - Effect.provideService( - CommandRunner.Service, - Context.get(runnerLayer, CommandRunner.Service), - ), - ); - yield* runtime.serve; - const failure = yield* shutdownOwner(runtime.access, true).pipe(Effect.flip); - expect(listCalls, failure.message).toBe(3); - expect(yield* (yield* stateFor(`${root}/state`)).read(saved.id)).toBeDefined(); - yield* Deferred.await(runtime.exit).pipe(Effect.timeout("10 seconds")); - const retryOwner = yield* Layer.build(ownerLayer).pipe( + const owner = yield* Layer.build(ownerLayer).pipe( Effect.map((context) => Context.get(context, Owner.Service)), ); - yield* retryOwner.namespace.destroy; - expect(listCalls).toBe(5); - expect(yield* (yield* stateFor(`${root}/state`)).read(saved.id)).toBeUndefined(); + yield* owner.namespace.destroy; + expect(yield* state.readClaims(saved.id)).toEqual([]); + expect(yield* state.read(saved.id)).toBeUndefined(); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); + +it.live( + "keeps a container claim recorded against a different daemon, never removing or dropping it", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-container-sweep-daemon-", + }); + const state = yield* stateFor(`${root}/state`); + const saved = { + id: "stack", + runtime: "docker" as const, + identity: { projectRoot: root, branchContext: "main", stackName: "sweep-daemon" }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + // Recorded against a daemon this reconcile run will never match. + yield* state.claim(saved.id, { + kind: "container", + id: containerId, + daemonId: "other-daemon", + }); + let rmCalls = 0; + const engine = ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command) || command.command !== "docker") + return Effect.die("Unexpected child process command"); + if (command.args[0] === "rm" && command.args.includes(containerId)) rmCalls++; + return Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(0), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + stdout: Stream.empty, + stderr: Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }), + ); + }); + const ownerLayer = Owner.layer({ + saved, + root: `${root}/data`, + cacheRoot: `${root}/cache`, + engineTarget, + }).pipe( + Layer.provide( + Layer.merge( + Layer.succeed(StackNamespace.Service, state), + Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, engine), + ), + ), + ); + const owner = yield* Layer.build(ownerLayer).pipe( + Effect.map((context) => Context.get(context, Owner.Service)), + ); + const failure = yield* owner.namespace.destroy.pipe(Effect.flip); + expect(rmCalls).toBe(0); + expect(failure.message).toContain(`container ${containerId} (daemon other-daemon)`); + expect(failure.message).toContain("Current daemon: current-daemon."); + expect(failure.message).toContain(`${root}/state/stack/claims.json`); + // Kept, not dropped: a full deregistration would otherwise have discarded it unreconciled. + expect(yield* state.read(saved.id)).toBeDefined(); + expect((yield* state.readClaims(saved.id)).map((claim) => claim.id)).toEqual([containerId]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live( + "keeps a container claim with no recorded daemon id, even once the current daemon is known", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-container-sweep-no-daemon-", + }); + const state = yield* stateFor(`${root}/state`); + const saved = { + id: "stack", + runtime: "docker" as const, + identity: { projectRoot: root, branchContext: "main", stackName: "sweep-no-daemon" }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + // No daemon id recorded at all, unlike a mismatched one: nothing to compare the current, + // successfully-resolved daemon against, so this is kept too, not removed by assuming it's + // the same daemon. + yield* state.claim(saved.id, { kind: "container", id: containerId }); + let rmCalls = 0; + const engine = ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command) || command.command !== "docker") + return Effect.die("Unexpected child process command"); + if (command.args[0] === "rm" && command.args.includes(containerId)) rmCalls++; + return Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(0), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + stdout: Stream.empty, + stderr: Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }), + ); + }); + const ownerLayer = Owner.layer({ + saved, + root: `${root}/data`, + cacheRoot: `${root}/cache`, + engineTarget, + }).pipe( + Layer.provide( + Layer.merge( + Layer.succeed(StackNamespace.Service, state), + Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, engine), + ), + ), + ); + const owner = yield* Layer.build(ownerLayer).pipe( + Effect.map((context) => Context.get(context, Owner.Service)), + ); + const failure = yield* owner.namespace.destroy.pipe(Effect.flip); + expect(rmCalls).toBe(0); + expect(failure.message).toContain(`container ${containerId} (daemon unknown)`); + expect(failure.message).toContain("Current daemon: current-daemon."); + expect(yield* state.read(saved.id)).toBeDefined(); + expect((yield* state.readClaims(saved.id)).map((claim) => claim.id)).toEqual([containerId]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts index 165ff7afb8..c3dd274d8e 100644 --- a/packages/stack/src/StackHost.integration.test.ts +++ b/packages/stack/src/StackHost.integration.test.ts @@ -36,6 +36,7 @@ import { bindControl, makeRuntime } from "./StackHost.ts"; import { shutdownOwner } from "../tests/owner.ts"; import { postgres } from "./Commands.ts"; import * as CommandRunner from "./host/CommandRunner.ts"; +import { noContainerClaims } from "../tests/claims.ts"; class HostTestError extends Data.TaggedError("HostTestError")<{ readonly message: string }> {} @@ -142,6 +143,7 @@ const inProcessRuntime = ( const acquired = yield* bindControl(); const toolContext = yield* Layer.build( CommandRunner.layer({ + claims: noContainerClaims, stackId: "stack", root, cacheRoot: "/tmp/supabase-stack-artifacts", diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index 5e3168ba41..12b24621ae 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -38,10 +38,12 @@ import { type HostEndpoint, type ShutdownFailure, } from "./HostProcess.ts"; +import { ChildProcessSpawner } from "effect/unstable/process"; import { projectSegmentFor } from "./identity/Identity.ts"; +import * as Claims from "./namespace/Claims.ts"; import * as Owner from "./Owner.ts"; import { StackError, stackError, StackRpc, type RunCommandPayload } from "./Rpc.ts"; -import { makeHostGateway } from "./runtime/Container.ts"; +import { engineUnreachable, makeHostGateway, resolveEngineTarget } from "./runtime/Container.ts"; import * as StackNamespace from "./StackNamespace.ts"; import { sweepOrphans } from "./Sweep.ts"; import { makeCommandAttachments } from "./host/CommandAttachments.ts"; @@ -433,14 +435,32 @@ export const runStackHost = Effect.fn("StackHost.run")( yield* fs.makeDirectory(dataRootPath, { recursive: true }); const dataRoot = yield* fs.realPath(dataRootPath); const project = projectSegmentFor(saved.identity, path); - yield* Owner.sweepContainers(saved, dataRoot).pipe( - Effect.mapError((cause) => - hostError( - "startup-cleanup", - cause, - cause.reason === "engine-unavailable" ? "runtime-unavailable" : undefined, - ), - ), + const asHostError = (cause: unknown) => + hostError( + "startup-cleanup", + cause, + cause instanceof Object && "reason" in cause && cause.reason === "engine-unavailable" + ? "runtime-unavailable" + : undefined, + ); + // Resolved once, here, for this owner's whole lifetime: the container runtime, the + // storage helpers, the host-gateway probes and reconcile below all share this one + // target instead of each resolving (and so potentially disagreeing on) their own. + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const engineTarget = + saved.runtime === "native" + ? undefined + : yield* resolveEngineTarget(spawner).pipe( + Effect.mapError((cause) => + hostError( + "startup-cleanup", + cause, + engineUnreachable(cause) ? "runtime-unavailable" : undefined, + ), + ), + ); + yield* Owner.sweepContainers(state, saved, dataRoot, engineTarget).pipe( + Effect.mapError(asHostError), ); const hostGateway = yield* makeHostGateway; const services = yield* Layer.build( @@ -450,6 +470,7 @@ export const runStackHost = Effect.fn("StackHost.run")( root: dataRoot, cacheRoot: options.cacheRoot, hostGateway, + engineTarget, }), CommandRunner.layer({ stackId: saved.id, @@ -457,7 +478,9 @@ export const runStackHost = Effect.fn("StackHost.run")( root: dataRoot, cacheRoot: options.cacheRoot, runtime: saved.runtime, + claims: Claims.forStack(state, saved.id).containers, hostGateway, + engineTarget, }), ).pipe(Layer.provide(Layer.succeed(StackNamespace.Service, state))), ); diff --git a/packages/stack/src/StackNamespace.ts b/packages/stack/src/StackNamespace.ts index bacf8ada1a..f750b0cf11 100644 --- a/packages/stack/src/StackNamespace.ts +++ b/packages/stack/src/StackNamespace.ts @@ -1,4 +1,5 @@ import { Context, Effect, FileSystem, Layer, Path } from "effect"; +import * as Claims from "./namespace/Claims.ts"; import { namespaceError, type NamespaceError } from "./namespace/Capabilities.ts"; import * as Lease from "./namespace/Lease.ts"; import * as Registry from "./namespace/Registry.ts"; @@ -15,7 +16,10 @@ export { type StackClaims, } from "./namespace/Registry.ts"; -export interface Interface extends Registry.Interface, Lease.Interface {} +export interface Interface extends Registry.Interface, Lease.Interface, Claims.Interface { + /** The namespace's own root directory, so a caller can name a stack's exact claims file. */ + readonly root: string; +} export class Service extends Context.Service()( "@supabase/stack/StackNamespace", @@ -52,7 +56,8 @@ const make = Effect.fn("Namespace.acquire")(function* ( onLeaseContended: options.onLeaseContended, isRegistered: (id) => registry.read(id).pipe(Effect.map((saved) => saved !== undefined)), }); - return { ...registry, ...lease }; + const claims = yield* Claims.make({ root, platform: options.platform }); + return { ...registry, ...lease, ...claims, root }; }); export const layer = (options: Options) => diff --git a/packages/stack/src/Sweep.integration.test.ts b/packages/stack/src/Sweep.integration.test.ts index 28a4c7536d..bc874b89ec 100644 --- a/packages/stack/src/Sweep.integration.test.ts +++ b/packages/stack/src/Sweep.integration.test.ts @@ -18,10 +18,11 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- test-only native watcher for synchronous attachment, no Effect wrapper gives this guarantee. import { watch as nodeWatch } from "node:fs"; import { launchHost } from "./HostProcess.ts"; -import { makeContainerRuntime } from "./runtime/Container.ts"; +import { makeContainerRuntime, resolveEngineTarget } from "./runtime/Container.ts"; import * as StackNamespace from "./StackNamespace.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; import { shutdownOwner, watchLeaseRelease } from "../tests/owner.ts"; +import { noContainerClaims } from "../tests/claims.ts"; class SweepTestError extends Data.TaggedError("SweepTestError")<{ readonly message: string }> {} @@ -170,7 +171,13 @@ it.live.skipIf(process.platform === "win32")( ); const rootA = path.dirname(path.dirname(dataA)); const cacheRoot = `${path.dirname(rootA)}/cache`; - const helper = yield* makeContainerRuntime({ engine: "docker", root: dataA }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const target = yield* resolveEngineTarget(spawner); + const helper = yield* makeContainerRuntime({ + claims: noContainerClaims, + target, + root: dataA, + }); yield* helper.prepare(helperImage); const state = Context.get( yield* Layer.build(StackNamespace.layer({ root: rootA })), @@ -181,6 +188,10 @@ it.live.skipIf(process.platform === "win32")( const dead = (yield* launchHost(state, { stateRoot: rootA, cacheRoot, stackId: deadId })) .endpoint; const orphan = yield* createOwnedContainer(deadId, dataA); + // The dead owner claimed this container before it was killed; the reconcile loop that + // replaces label-based sweeping only ever acts on recorded claims, never on labels alone. + // Recorded against this real daemon's own id: a claim with none is now always kept. + yield* state.claim(deadId, { kind: "container", id: orphan, daemonId: target.daemonId }); const otherRoot = yield* createOwnedContainer(deadId, dataB); const deadReleased = yield* watchLeaseRelease(rootA, deadId); yield* Effect.sync(() => process.kill(dead.pid, "SIGKILL")); diff --git a/packages/stack/src/Sweep.ts b/packages/stack/src/Sweep.ts index 05e017ab51..556b114149 100644 --- a/packages/stack/src/Sweep.ts +++ b/packages/stack/src/Sweep.ts @@ -1,6 +1,8 @@ import { Context, DateTime, Effect, FileSystem, Layer, Path } from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; import { sweepTimeout } from "./HostProcess.ts"; import * as Owner from "./Owner.ts"; +import { resolveEngineTarget } from "./runtime/Container.ts"; import * as StackNamespace from "./StackNamespace.ts"; /** Runs a dead session stack's own destroy path in this process while holding its lease. */ @@ -11,9 +13,11 @@ const destroyStack = Effect.fn("Sweep.destroyStack")(function* ( cacheRoot: string, ) { const fs = yield* FileSystem.FileSystem; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; yield* fs.makeDirectory(dataRoot, { recursive: true }); + const engineTarget = saved.runtime === "native" ? undefined : yield* resolveEngineTarget(spawner); const context = yield* Layer.build( - Owner.layer({ saved, root: dataRoot, cacheRoot }).pipe( + Owner.layer({ saved, root: dataRoot, cacheRoot, engineTarget }).pipe( Layer.provide(Layer.succeed(StackNamespace.Service, state)), ), ); @@ -52,7 +56,12 @@ export const reclaimStack = Effect.fn("Sweep.reclaimStack")(function* (options: const dataRoot = path.join(yield* fs.realPath(options.stateRoot), id, "data"); if (saved.lifetime === "session") yield* destroyStack(state, saved, dataRoot, options.cacheRoot); - else yield* Owner.sweepContainers(saved, dataRoot); + else { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const engineTarget = + saved.runtime === "native" ? undefined : yield* resolveEngineTarget(spawner); + yield* Owner.sweepContainers(state, saved, dataRoot, engineTarget); + } return true; }), ).pipe(Effect.timeout(sweepTimeout)); diff --git a/packages/stack/src/create.owner-rollback.integration.test.ts b/packages/stack/src/create.owner-rollback.integration.test.ts index 34ab4ff186..c038be8ad8 100644 --- a/packages/stack/src/create.owner-rollback.integration.test.ts +++ b/packages/stack/src/create.owner-rollback.integration.test.ts @@ -1,12 +1,32 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Effect, Exit, Fiber, FileSystem, Layer, Path } from "effect"; +import { Context, Effect, Exit, Fiber, FileSystem, Layer, Path } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { create, discover } from "./effect.ts"; import { deriveStackId, resolveStackIdentity } from "./identity/Identity.ts"; +import * as StackNamespace from "./StackNamespace.ts"; const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); +/** Seeds a container claim for a not-yet-created stack id, as an orphaned claim would be found. */ +const seedContainerClaim = ( + stateRoot: string, + id: string, + containerId: string, + daemonId?: string, +) => + Effect.scoped( + Layer.build(StackNamespace.layer({ root: stateRoot })).pipe( + Effect.flatMap((context) => + Context.get(context, StackNamespace.Service).claim(id, { + kind: "container", + id: containerId, + ...(daemonId === undefined ? {} : { daemonId }), + }), + ), + ), + ); + it.live( "removes a stack it just registered when the owner fails to launch, and lets a retry succeed", () => @@ -19,7 +39,7 @@ it.live( const dockerShim = path.join(binDir, "docker"); yield* fs.writeFileString( dockerShim, - "#!/bin/sh\necho 'docker daemon unreachable' >&2\nexit 1\n", + "#!/bin/sh\necho 'Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?' >&2\nexit 1\n", ); yield* fs.chmod(dockerShim, 0o755); // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the detached host subprocess inherits PATH; this is not application config. @@ -41,13 +61,18 @@ it.live( } satisfies Parameters[0]; const identity = yield* resolveStackIdentity(options); const id = yield* deriveStackId(identity); + // An orphaned claim from a never-fully-registered prior attempt, found by this owner's + // startup reconcile before it finishes registering. + yield* seedContainerClaim(options.stateRoot, id, "rollback0000000000000000000000001"); const launchFailure = yield* Effect.flip(create({ ...options, startOwner: true })); - expect(launchFailure.message).toContain("docker daemon unreachable"); + expect(launchFailure.message).toContain("Cannot connect to the Docker daemon"); expect(yield* discover({ stateRoot: options.stateRoot })).toEqual([]); - const stackDirExit = yield* Effect.exit(fs.access(path.join(options.stateRoot, id))); - expect(Exit.isFailure(stackDirExit)).toBe(true); + const stateFileExit = yield* Effect.exit( + fs.access(path.join(options.stateRoot, id, "state.json")), + ); + expect(Exit.isFailure(stateFileExit)).toBe(true); const retried = yield* create({ ...options, runtime: "native" }); expect(retried.id).toBe(id); @@ -60,7 +85,7 @@ it.live("removes a stack it just registered when its owner launch is interrupted const path = yield* Path.Path; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-create-interrupt-" }); - // The shim signals through a FIFO once the owner's startup sweep is running, then blocks. + // The shim signals through a FIFO once the owner's startup reconcile is running, then blocks. const started = path.join(root, "sweep-started"); yield* Effect.scoped( spawner @@ -72,17 +97,29 @@ it.live("removes a stack it just registered when its owner launch is interrupted const dockerShim = path.join(binDir, "docker"); yield* fs.writeFileString( dockerShim, - `#!/bin/sh\nif [ "$1" = "ps" ]; then echo started > '${started}'; exec sleep 60; fi\nexit 0\n`, + // The leading check strips the pinned `--host ` prefix every invocation now + // carries (fixed below via DOCKER_HOST), so "$1" below still sees the real command. + `#!/bin/sh\nif [ "$1" = "--host" ]; then shift 2; fi\nif [ "$1" = "rm" ]; then echo started > '${started}'; exec sleep 60; fi\nif [ "$1" = "info" ]; then echo fake-daemon-id; fi\nexit 0\n`, ); yield* fs.chmod(dockerShim, 0o755); // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the detached host subprocess inherits PATH; this is not application config. const originalPath = process.env.PATH; // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. process.env.PATH = `${binDir}:${originalPath ?? ""}`; + // Fixes the pinned endpoint to a dummy socket, so resolving it never needs `context show`/ + // `context inspect`, which this minimal shim doesn't answer. + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. + const originalDockerHost = process.env.DOCKER_HOST; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. + process.env.DOCKER_HOST = "unix:///var/run/shimmed-docker.sock"; yield* Effect.addFinalizer(() => Effect.sync(() => { // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. process.env.PATH = originalPath; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. + if (originalDockerHost === undefined) delete process.env.DOCKER_HOST; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. + else process.env.DOCKER_HOST = originalDockerHost; }), ); @@ -92,6 +129,16 @@ it.live("removes a stack it just registered when its owner launch is interrupted cacheRoot: `${root}/cache`, runtime: "docker", } satisfies Parameters[0]; + const identity = yield* resolveStackIdentity(options); + const id = yield* deriveStackId(identity); + // Recorded against the daemon this reconcile run will match: the engine is reachable and + // identifiable here, so the interrupted launch's own rollback is what's under test. + yield* seedContainerClaim( + options.stateRoot, + id, + "interrupt0000000000000000000000001", + "fake-daemon-id", + ); const creating = yield* create({ ...options, startOwner: true }).pipe(Effect.forkChild); yield* fs.readFileString(started); yield* Fiber.interrupt(creating); diff --git a/packages/stack/src/destroy.runtime-unavailable.integration.test.ts b/packages/stack/src/destroy.runtime-unavailable.integration.test.ts index 595b75a6a5..b32a78ea29 100644 --- a/packages/stack/src/destroy.runtime-unavailable.integration.test.ts +++ b/packages/stack/src/destroy.runtime-unavailable.integration.test.ts @@ -1,37 +1,81 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Effect, Exit, FileSystem, Layer, Path } from "effect"; -import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { Context, Effect, FileSystem, Layer, Path } from "effect"; import { create, discover } from "./effect.ts"; +import * as StackNamespace from "./StackNamespace.ts"; const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); -const shimDocker = Effect.fn("shimDocker")(function* ( - root: string, - script: string, - engine: "docker" | "podman" = "docker", -) { +// Strips the pinned `--host ` prefix every invocation now carries, so a shim's own +// `"$1"` checks see the real command unchanged; `DOCKER_HOST` below makes that prefix constant. +const stripPinPrefix = 'if [ "$1" = "--host" ]; then shift 2; fi\n'; + +const shimDocker = Effect.fn("shimDocker")(function* (root: string, script: string) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const binDir = path.join(root, "bin"); yield* fs.makeDirectory(binDir, { recursive: true }); - const dockerShim = path.join(binDir, engine); - yield* fs.writeFileString(dockerShim, script); + const dockerShim = path.join(binDir, "docker"); + yield* fs.writeFileString( + dockerShim, + script.replace("#!/bin/sh\n", `#!/bin/sh\n${stripPinPrefix}`), + ); yield* fs.chmod(dockerShim, 0o755); // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the detached host subprocess inherits PATH; this is not application config. const originalPath = process.env.PATH; // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. process.env.PATH = `${binDir}:${originalPath ?? ""}`; + // Fixes the pinned endpoint to a dummy socket, so resolving it never needs `context show`/ + // `context inspect`, which these minimal shims don't answer. + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. + const originalDockerHost = process.env.DOCKER_HOST; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. + process.env.DOCKER_HOST = "unix:///var/run/shimmed-docker.sock"; yield* Effect.addFinalizer(() => Effect.sync(() => { // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. process.env.PATH = originalPath; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. + if (originalDockerHost === undefined) delete process.env.DOCKER_HOST; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. + else process.env.DOCKER_HOST = originalDockerHost; }), ); }); +/** Opens the real namespace at `stateRoot`, bypassing any owner, to seed or read claims directly. */ +const withNamespace = ( + stateRoot: string, + use: (state: StackNamespace.Interface) => Effect.Effect, +) => + Effect.scoped( + Layer.build(StackNamespace.layer({ root: stateRoot })).pipe( + Effect.flatMap((context) => use(Context.get(context, StackNamespace.Service))), + ), + ); + +/** Records a container claim as if a now-dead owner had created it before being interrupted. */ +const seedContainerClaim = ( + stateRoot: string, + id: string, + containerId: string, + daemonId?: string, +) => + withNamespace(stateRoot, (state) => + state.claim(id, { + kind: "container", + id: containerId, + ...(daemonId === undefined ? {} : { daemonId }), + }), + ); + +const readClaimIds = (stateRoot: string, id: string) => + withNamespace(stateRoot, (state) => state.readClaims(id)).pipe( + Effect.map((claims) => claims.map((claim) => claim.id)), + ); + it.live( - "removes a stack's registration and data when destroy finds no owner and its engine is unreachable", + "keeps a stack's registration and claims when destroy finds no owner and its engine is unreachable, then finishes the cleanup once a later owner can start", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -48,36 +92,35 @@ it.live( runtime: "docker", } satisfies Parameters[0]; const stack = yield* create(options); - const dataRoot = `${options.stateRoot}/${stack.id}/data`; - yield* fs.makeDirectory(`${dataRoot}/db-instance`, { recursive: true }); - yield* fs.writeFileString( - `${dataRoot}/db-instance/.supabase-database-storage.json`, - `{"backend":"docker","volume":"supabase-db-0123456789abcdef","namespace":"instance-${stack.id}-db-instance","cacheNamespace":"cache-${"0".repeat(32)}","daemonId":"daemon","initialized":true}`, + // Recorded against the daemon the engine will report once it is back: the second destroy + // attempt below, once the engine is reachable again, is expected to finish the cleanup. + yield* seedContainerClaim( + options.stateRoot, + stack.id, + "offline0000000000000000000000000001", + "fake-daemon-id", ); - const resolvedDataRoot = yield* fs.realPath(dataRoot); const result = yield* stack.destroy; - expect(result).toEqual({ - runtimeCleanup: "skipped", - engine: "docker", - cleanupCommands: [ - `sh -c 'ids=$(docker ps --all --quiet --no-trunc --filter '\\''label=com.supabase.stack=${stack.id}'\\'' --filter '\\''label=com.supabase.stack-root=${resolvedDataRoot}'\\'') && { [ -z "$ids" ] || docker rm --force $ids; }'`, - expect.stringMatching( - new RegExp( - `^docker run --rm --mount 'type=volume,src=supabase-db-0123456789abcdef,dst=/store' '[^']+' /bin/sh -c 'rm -rf /store/instance-${stack.id}-db-instance'$`, - "u", - ), - ), - ], - }); + expect(result).toEqual({ runtimeCleanup: "skipped", engine: "docker" }); + expect(yield* discover({ stateRoot: options.stateRoot })).toHaveLength(1); + expect(yield* readClaimIds(options.stateRoot, stack.id)).toEqual([ + "offline0000000000000000000000000001", + ]); + // The engine is back: destroy retried finishes the exact claimed cleanup automatically. + yield* shimDocker( + root, + '#!/bin/sh\nif [ "$1" = "info" ]; then\n echo \'fake-daemon-id\'\nfi\nexit 0\n', + ); + const second = yield* stack.destroy; + expect(second).toEqual({ runtimeCleanup: "complete" }); expect(yield* discover({ stateRoot: options.stateRoot })).toEqual([]); - const stackDirExit = yield* Effect.exit(fs.access(`${options.stateRoot}/${stack.id}`)); - expect(Exit.isFailure(stackDirExit)).toBe(true); }).pipe(Effect.scoped, Effect.provide(layer)), + 30_000, ); -it.live("removes a stack offline when Windows reports its Docker daemon pipe is missing", () => +it.live("keeps a stack registered when Windows reports its Docker daemon pipe is missing", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-windows-" }); @@ -92,15 +135,16 @@ it.live("removes a stack offline when Windows reports its Docker daemon pipe is cacheRoot: `${root}/cache`, runtime: "docker", }); + yield* seedContainerClaim(stateRoot, stack.id, "windows000000000000000000000000001"); const result = yield* stack.destroy; expect(result.runtimeCleanup).toBe("skipped"); - expect(yield* discover({ stateRoot })).toEqual([]); + expect(yield* discover({ stateRoot })).toHaveLength(1); }).pipe(Effect.scoped, Effect.provide(layer)), ); -it.live("removes a stack offline when its engine CLI is not installed", () => +it.live("keeps a stack registered when its engine CLI is not installed", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-no-cli-" }); @@ -111,6 +155,7 @@ it.live("removes a stack offline when its engine CLI is not installed", () => cacheRoot: `${root}/cache`, runtime: "docker", }); + yield* seedContainerClaim(stateRoot, stack.id, "nocli00000000000000000000000000001"); yield* fs.makeDirectory(`${root}/empty-bin`); // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the detached host subprocess inherits PATH; this is not application config. const originalPath = process.env.PATH; @@ -126,11 +171,11 @@ it.live("removes a stack offline when its engine CLI is not installed", () => const result = yield* stack.destroy; expect(result.runtimeCleanup).toBe("skipped"); - expect(yield* discover({ stateRoot })).toEqual([]); + expect(yield* discover({ stateRoot })).toHaveLength(1); }).pipe(Effect.scoped, Effect.provide(layer)), ); -it.live("removes a stack offline when Docker reports its API socket is missing", () => +it.live("keeps a stack registered when Docker reports its API socket is missing", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-missing-socket-" }); @@ -145,111 +190,15 @@ it.live("removes a stack offline when Docker reports its API socket is missing", cacheRoot: `${root}/cache`, runtime: "docker", }); + yield* seedContainerClaim(stateRoot, stack.id, "socket0000000000000000000000000001"); const result = yield* stack.destroy; expect(result.runtimeCleanup).toBe("skipped"); - expect(yield* discover({ stateRoot })).toEqual([]); - }).pipe(Effect.scoped, Effect.provide(layer)), -); - -it.live( - "prints a container cleanup command that fails while the engine is down and removes every listed container once it is back", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-command-" }); - const unreachable = - "#!/bin/sh\necho 'Cannot connect to the Docker daemon at tcp://127.0.0.1:1. Is the docker daemon running?' >&2\nexit 1\n"; - yield* shimDocker(root, unreachable); - const stack = yield* create({ - projectRoot: root, - stateRoot: `${root}/state`, - cacheRoot: `${root}/cache`, - runtime: "docker", - }); - const result = yield* stack.destroy; - if (result.runtimeCleanup !== "skipped") return yield* Effect.die("cleanup was not skipped"); - const [containerCommand] = result.cleanupCommands; - if (containerCommand === undefined) return yield* Effect.die("container command missing"); - const run = Effect.scoped( - spawner - .spawn(ChildProcess.make("/bin/sh", ["-c", containerCommand])) - .pipe(Effect.flatMap((child) => child.exitCode)), - ); - - expect(Number(yield* run)).not.toBe(0); - yield* fs.writeFileString( - `${root}/bin/docker`, - '#!/bin/sh\nif [ "$1" = "ps" ]; then exit 0; fi\necho "docker rm requires at least 1 argument" >&2\nexit 1\n', - ); - expect(Number(yield* run)).toBe(0); - // Removal succeeds only when both listed IDs arrive as separate arguments. - yield* fs.writeFileString( - `${root}/bin/docker`, - '#!/bin/sh\nif [ "$1" = "ps" ]; then printf "aaa111\\nbbb222\\n"; exit 0; fi\nif [ "$1" = "rm" ] && [ "$2" = "--force" ] && [ "$#" -eq 4 ] && [ "$3" = "aaa111" ] && [ "$4" = "bbb222" ]; then exit 0; fi\necho "unexpected arguments: $*" >&2\nexit 1\n', - ); - expect(Number(yield* run)).toBe(0); - }).pipe(Effect.scoped, Effect.provide(layer)), -); - -it.live("keeps a stack registered when Podman rejects its credentials", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-podman-auth-" }); - yield* shimDocker( - root, - "#!/bin/sh\necho 'Cannot connect to Podman. Please verify your connection to the Linux system using `podman system connection list`, or try `podman machine init` and `podman machine start` to manage a new Linux VM' >&2\necho 'Error: unable to connect to Podman socket: ssh: handshake failed: ssh: unable to authenticate, attempted methods [none publickey], no supported methods remain' >&2\nexit 125\n", - "podman", - ); - const stateRoot = `${root}/state`; - const stack = yield* create({ - projectRoot: root, - stateRoot, - cacheRoot: `${root}/cache`, - runtime: "podman", - }); - - const destroyFailure = yield* Effect.flip(stack.destroy); - - expect(destroyFailure.message).toContain("unable to authenticate"); expect(yield* discover({ stateRoot })).toHaveLength(1); }).pipe(Effect.scoped, Effect.provide(layer)), ); -it.live.skipIf(process.getuid?.() === 0)( - "keeps a stack and its data when offline destroy cannot remove container-owned host data", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-host-data-" }); - yield* shimDocker( - root, - "#!/bin/sh\necho 'Cannot connect to the Docker daemon at tcp://127.0.0.1:1. Is the docker daemon running?' >&2\nexit 1\n", - ); - const stateRoot = `${root}/state`; - const stack = yield* create({ - projectRoot: root, - stateRoot, - cacheRoot: `${root}/cache`, - runtime: "docker", - }); - // Stands in for PostgreSQL files a container wrote as its own user. - const instanceRoot = `${stateRoot}/${stack.id}/data/db-instance`; - yield* fs.makeDirectory(`${instanceRoot}/data`, { recursive: true }); - yield* fs.writeFileString(`${instanceRoot}/owned-file`, "owned data"); - yield* fs.chmod(`${instanceRoot}/data`, 0o000); - yield* Effect.addFinalizer(() => fs.chmod(`${instanceRoot}/data`, 0o755).pipe(Effect.ignore)); - - const destroyFailure = yield* Effect.flip(stack.destroy); - - expect(destroyFailure.message).toContain("start Docker and run destroy again"); - expect(yield* discover({ stateRoot })).toHaveLength(1); - expect(yield* fs.exists(`${instanceRoot}/owned-file`)).toBe(true); - }).pipe(Effect.scoped, Effect.provide(layer)), -); - it.live("keeps a stack registered when its container engine rejects the listing", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -266,11 +215,17 @@ it.live("keeps a stack registered when its container engine rejects the listing" runtime: "docker", } satisfies Parameters[0]; const stack = yield* create(options); + yield* seedContainerClaim(options.stateRoot, stack.id, "permission00000000000000000000001"); const destroyFailure = yield* Effect.flip(stack.destroy); + // The socket itself refuses the owner's own startup, resolving its pinned engine target: + // the owner never starts, so the stack stays registered for a later retry. expect(destroyFailure.message).toContain("permission denied"); expect(yield* discover({ stateRoot: options.stateRoot })).toHaveLength(1); + expect(yield* readClaimIds(options.stateRoot, stack.id)).toEqual([ + "permission00000000000000000000001", + ]); }).pipe(Effect.scoped, Effect.provide(layer)), ); @@ -282,14 +237,14 @@ it.live("keeps a stack registered when its container engine is reachable but cle root, [ "#!/bin/sh", - 'if [ "$1" = "ps" ]; then', - ' echo "abc123def456"', - " exit 0", - "fi", 'if [ "$1" = "rm" ]; then', " echo 'docker: Error response from daemon: container removal failed' >&2", " exit 1", "fi", + // The daemon is reachable and identifiable; only the targeted removal fails. + 'if [ "$1" = "info" ]; then', + " echo 'fake-daemon-id'", + "fi", "exit 0", "", ].join("\n"), @@ -302,10 +257,22 @@ it.live("keeps a stack registered when its container engine is reachable but cle runtime: "docker", } satisfies Parameters[0]; const stack = yield* create(options); + // Recorded against the daemon this reconcile run will match: the engine is reachable and + // identifiable here, so only the targeted removal itself is expected to fail. + yield* seedContainerClaim( + options.stateRoot, + stack.id, + "cleanupfail0000000000000000000001", + "fake-daemon-id", + ); const destroyFailure = yield* Effect.flip(stack.destroy); expect(destroyFailure.message).toContain("container removal failed"); expect(yield* discover({ stateRoot: options.stateRoot })).toHaveLength(1); + // Cleanup authority never transferred away from the claim, so a later retry can still finish it. + expect(yield* readClaimIds(options.stateRoot, stack.id)).toEqual([ + "cleanupfail0000000000000000000001", + ]); }).pipe(Effect.scoped, Effect.provide(layer)), ); diff --git a/packages/stack/src/effect.integration.test.ts b/packages/stack/src/effect.integration.test.ts index 3a50b3f98c..6991e864b5 100644 --- a/packages/stack/src/effect.integration.test.ts +++ b/packages/stack/src/effect.integration.test.ts @@ -9,7 +9,6 @@ import { Layer, Option, Redacted, - Schema, Scope, Stream, } from "effect"; @@ -37,9 +36,6 @@ import { deriveStackId, resolveStackIdentity } from "./identity/Identity.ts"; const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); // Below every OS ephemeral range, so another test's outbound socket cannot already hold it. const FIXED_API_PORT = 24_393; -const databaseOwnerMarker = Schema.fromJsonString( - Schema.Struct({ stackId: Schema.String, instanceId: Schema.String }), -); it.live("registers and discovers saved definitions without inventing live observations", () => Effect.gen(function* () { @@ -175,12 +171,11 @@ it.live( }, }); const dataRoot = `${options.stateRoot}/${stack.id}/data/${instance.id}`; - const marker = `${dataRoot}/.supabase-database-owner.json`; - yield* fs.makeDirectory(dataRoot, { recursive: true }); - const writeMarker = Schema.encodeEffect(databaseOwnerMarker); - yield* writeMarker({ stackId: "another-stack", instanceId: instance.id }).pipe( - Effect.flatMap((value) => fs.writeFileString(marker, value)), - ); + // Ownership is by location: a root that is a symlink (not a real owned directory, perhaps + // tampered with) is refused rather than traversed or removed. + yield* fs.remove(dataRoot, { recursive: true, force: true }); + const outside = yield* fs.makeTempDirectoryScoped({ prefix: "destroy-failure-outside-" }); + yield* fs.symlink(outside, dataRoot); const running = yield* discover(options); expect(running).toHaveLength(1); expect(running[0]?.host).toBeDefined(); @@ -196,19 +191,17 @@ it.live( { id: instance.id, succeeded: false, - error: expect.stringContaining("Database root belongs to another instance"), + error: expect.stringContaining("is a symlink"), }, ], }); } - expect(yield* fs.readFileString(marker)).toContain("another-stack"); + expect(yield* fs.readLink(dataRoot).pipe(Effect.isSuccess)).toBe(true); const retained = yield* discover(options); expect(retained).toHaveLength(1); expect(retained[0]?.host).toBeUndefined(); - yield* writeMarker({ stackId: stack.id, instanceId: instance.id }).pipe( - Effect.flatMap((value) => fs.writeFileString(marker, value)), - ); + yield* fs.remove(dataRoot, { force: true }); yield* stack.destroy; expect(yield* discover(options)).toHaveLength(0); }), diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index b30e4d4706..a82ba39fe5 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -2,7 +2,6 @@ import { Cause, Context, Crypto, - DateTime, Deferred, Effect, Exit, @@ -38,8 +37,6 @@ import { type PlannedInstance, type SupabaseCompositionOptions, } from "./composition/Supabase.ts"; -import { removeStackContainersCommand } from "./runtime/Container.ts"; -import { volumeDataCleanupCommands } from "./storage/DockerDatabaseStorage.ts"; import { deriveStackId, resolveStackIdentity } from "./identity/Identity.ts"; import { failureMessage } from "./internal/failure-message.ts"; import * as StackNamespace from "./StackNamespace.ts"; @@ -96,7 +93,7 @@ export interface StackLocations { export interface CreateOptions extends StackLocations { readonly projectRoot: string; readonly name?: string; - readonly runtime: "native" | "docker" | "podman"; + readonly runtime: "native" | "docker"; /** * A `session` stack starts its owner at creation and is destroyed when the creating handle's * scope closes or its process exits; a `detached` stack (the default) outlives its creator. @@ -188,18 +185,14 @@ export type ServiceInstances = { }; type AnyInstance = ServiceInstances[Kind]; /** - * The outcome of {@link Stack.destroy}. `skipped` means the stack's registration and host data - * were removed without its container engine, because the engine was unreachable; its containers - * and any database data in engine volumes remain, and `cleanupCommands` remove them once the - * engine is running. + * The outcome of {@link Stack.destroy}. `skipped` means the stack's container engine was + * unreachable, so nothing was confirmed removed: its registration, data and claimed resources all + * stay in place. Recovery means calling destroy again once the engine is reachable; reopening the + * stack does not clean this up on its own. */ export type DestroyResult = | { readonly runtimeCleanup: "complete" } - | { - readonly runtimeCleanup: "skipped"; - readonly engine: "docker" | "podman"; - readonly cleanupCommands: ReadonlyArray; - }; + | { readonly runtimeCleanup: "skipped"; readonly engine: "docker" }; /** Options for streaming PostgreSQL command input and output. */ export interface PostgresCommandOptions { readonly args?: ReadonlyArray; @@ -302,85 +295,26 @@ interface Connection { /** Replaced by a newer connection; closes once its last user finishes. */ retired: boolean; } -/** Finds a directory below `directory` that the current user cannot empty and delete. */ -const firstUnremovableDirectory = ( - fs: FileSystem.FileSystem, - path: Path.Path, - directory: string, -): Effect.Effect => - Effect.gen(function* () { - const entries = yield* fs.readDirectory(directory).pipe(Effect.option); - const writable = yield* fs.access(directory, { writable: true }).pipe(Effect.isSuccess); - if (Option.isNone(entries) || !writable) return directory; - for (const entry of entries.value) { - const child = path.join(directory, entry); - const info = yield* fs.stat(child).pipe(Effect.option); - if (Option.isNone(info) || info.value.type !== "Directory") continue; - if (yield* fs.readLink(child).pipe(Effect.isSuccess)) continue; - const blocked = yield* firstUnremovableDirectory(fs, path, child); - if (blocked !== undefined) return blocked; - } - return undefined; - }); - /** - * Destroys a stack whose owner cannot start because its container engine is unreachable: under - * the stack's lease it removes the registration and host data, and returns the commands that - * remove the containers and engine-volume data left behind. + * Destroys a stack whose owner cannot start because its container engine is unreachable: since + * nothing can be confirmed removed, registration, data and claims all stay exactly as they are. + * The next lease acquisition's reconcile loop (and, once an owner can start, the stack's normal + * destroy path) finishes the cleanup once the engine is reachable again. */ -const destroyWithoutEngine = Effect.fn("Stack.destroyWithoutEngine")(function* ( - state: StackNamespace.Interface, - saved: SavedStack, - locations: StackLocations, - engine: "docker" | "podman", -) { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const id = saved.id; - const dataRoot = path.join(locations.stateRoot, id, "data"); - return yield* Effect.scoped( - Effect.gen(function* () { - const lease = yield* state - .acquireLease(id) - .pipe( - Effect.catchTag("Namespace.LeaseHeldError", () => - failure("destroy", "An owner for this stack started during destroy; run destroy again"), - ), - ); - yield* Effect.addFinalizer(() => lease.retractHolder.pipe(Effect.ignore)); - yield* lease.publishHolder({ - role: "sweeper", - pid: process.pid, - startedAt: DateTime.formatIso(yield* DateTime.now), - }); - const current = yield* state.read(id); - // Container-written host data, such as database files below Docker 26, can belong to the - // container user; only the engine can delete it, so refuse before deleting anything. - const blocked = - current !== undefined && (yield* fs.exists(dataRoot)) - ? yield* firstUnremovableDirectory(fs, path, dataRoot) - : undefined; - if (blocked !== undefined) { - const engineName = engine === "docker" ? "Docker" : "Podman"; - return yield* failure( - "destroy", - `Stack data at ${blocked} can only be removed by ${engineName}; start ${engineName} and run destroy again`, - ); - } - // Containers are labelled with the resolved data root the owner ran with. - const root = yield* fs.realPath(dataRoot).pipe(Effect.orElseSucceed(() => dataRoot)); - const cleanupCommands = [ - removeStackContainersCommand({ engine, stackId: id, root }), - ...(yield* volumeDataCleanupCommands({ engine, root, fs, path })), - ]; - if (current !== undefined) { - yield* fs.remove(dataRoot, { recursive: true, force: true }); - yield* state.withLock(state.remove(id)); - } - return { runtimeCleanup: "skipped", engine, cleanupCommands } as const; - }), - ).pipe(Effect.mapError((cause) => failure("destroy", cause))); -}); +const destroyWithoutEngine = Effect.fn("Stack.destroyWithoutEngine")( + function* (state: StackNamespace.Interface, saved: SavedStack, engine: "docker") { + const id = saved.id; + if (yield* state.leased(id)) + return yield* failure( + "destroy", + "An owner for this stack started during destroy; run destroy again", + ); + const current = yield* state.read(id); + if (current === undefined) return { runtimeCleanup: "complete" } as const; + return { runtimeCleanup: "skipped", engine } as const; + }, + Effect.mapError((cause) => failure("destroy", cause)), +); /** `launch` starts an owner when none is live; `attach` requires a live one. */ type Reach = "launch" | "attach"; @@ -590,7 +524,7 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( Effect.mapError((cause) => failure(operation, cause)), ); if (engineUnavailable && engine !== undefined) - return yield* destroyWithoutEngine(state, saved, locations, engine).pipe( + return yield* destroyWithoutEngine(state, saved, engine).pipe( Effect.provideContext(services), ); if (endpoint === undefined) return { runtimeCleanup: "complete" } as const; diff --git a/packages/stack/src/functions/FunctionsBootstrap.ts b/packages/stack/src/functions/FunctionsBootstrap.ts index 340deed73b..0615744cc2 100644 --- a/packages/stack/src/functions/FunctionsBootstrap.ts +++ b/packages/stack/src/functions/FunctionsBootstrap.ts @@ -1,6 +1,4 @@ import { Crypto, Data, Effect, FileSystem, Path, PlatformError, Schema } from "effect"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no non-recursive directory removal operation. -import { rmdir } from "node:fs/promises"; import { StackIdSchema, type StackId } from "../identity/StackId.ts"; export class FunctionsBootstrapError extends Data.TaggedError("FunctionsBootstrapError")<{ @@ -14,8 +12,6 @@ export interface FunctionsBootstrapOwner { readonly write: (input: { readonly content: string; }) => Effect.Effect; - /** Removes only this stack's functions bootstrap root. */ - readonly cleanupAll: Effect.Effect; } export interface FunctionsBootstrapOwnerOptions { @@ -46,22 +42,6 @@ export const makeFunctionsBootstrapOwner = Effect.fn("FunctionsBootstrap.makeOwn if (!/^[a-zA-Z0-9_-]+$/u.test(options.instanceId)) return yield* failure("Invalid Functions instance identity"); const root = path.join(options.root, options.instanceId, "runtime", "functions"); - const removeEmptyDirectory = (directory: string) => - Effect.tryPromise({ - try: () => rmdir(directory), - catch: (cause) => - failure("Unable to clean Functions parent directory", { path: directory, cause }), - }).pipe( - Effect.catch((cause) => { - const code = - typeof cause.cause === "object" && cause.cause !== null && "code" in cause.cause - ? cause.cause.code - : undefined; - return code === "ENOENT" || code === "ENOTEMPTY" || code === "EEXIST" - ? Effect.void - : Effect.fail(cause); - }), - ); const write = Effect.fn("FunctionsBootstrap.write")(function* (input: { readonly content: string; @@ -124,14 +104,5 @@ export const makeFunctionsBootstrapOwner = Effect.fn("FunctionsBootstrap.makeOwn }); }); - const cleanupAll = mapFs( - root, - "clean functions bootstrap files", - fs.remove(root, { recursive: true, force: true }), - ).pipe( - Effect.andThen(removeEmptyDirectory(path.dirname(root))), - Effect.andThen(removeEmptyDirectory(path.dirname(path.dirname(root)))), - Effect.withSpan("FunctionsBootstrap.cleanupAll"), - ); - return { write, cleanupAll }; + return { write }; }); diff --git a/packages/stack/src/functions/functions-bootstrap.integration.test.ts b/packages/stack/src/functions/functions-bootstrap.integration.test.ts index cb1e008534..fd0c43b084 100644 --- a/packages/stack/src/functions/functions-bootstrap.integration.test.ts +++ b/packages/stack/src/functions/functions-bootstrap.integration.test.ts @@ -32,50 +32,6 @@ describe("functions bootstrap owner", () => { }).pipe(Effect.provide(NodeServices.layer)), ); - it.live("cleans one Functions bootstrap while preserving another instance", () => - Effect.gen(function* () { - const path = yield* Path.Path; - const { fs, root, stackId, owner } = yield* setupBootstrapOwner( - "stack-functions-bootstrap-cleanup-", - ); - const target = yield* owner.write({ content: "export default 1" }); - const other = yield* makeFunctionsBootstrapOwner({ - root, - stackId, - instanceId: "functions-two", - }); - const otherTarget = yield* other.write({ content: "export default 2" }); - - expect(yield* fs.exists(target)).toBe(true); - - yield* owner.cleanupAll; - - expect(yield* fs.exists(target)).toBe(false); - expect(yield* fs.readFileString(otherTarget)).toBe("export default 2"); - expect(yield* fs.exists(path.join(root, "functions-one", "runtime", "functions"))).toBe( - false, - ); - expect(yield* fs.exists(path.join(root, "functions-one"))).toBe(false); - expect(yield* fs.exists(path.join(root, "functions-two"))).toBe(true); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("recreates readable Functions bootstrap content after cleanup", () => - Effect.gen(function* () { - const { fs, owner } = yield* setupBootstrapOwner("stack-functions-bootstrap-recreate-"); - const first = yield* owner.write({ content: "export default 1" }); - - expect(yield* fs.exists(first)).toBe(true); - - yield* owner.cleanupAll; - - expect(yield* fs.exists(first)).toBe(false); - const recreated = yield* owner.write({ content: "export default 2" }); - - expect(yield* fs.readFileString(recreated)).toBe("export default 2"); - }).pipe(Effect.provide(NodeServices.layer)), - ); - it.live("returns the canonical published path for a symlinked state root", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -105,11 +61,6 @@ describe("functions bootstrap owner", () => { expect(target).toBe(expected); expect(yield* fs.readFileString(target)).toBe("export default 2"); expect(((yield* fs.stat(target)).mode ?? 0) & 0o777).toBe(0o600); - - yield* owner.cleanupAll; - expect( - yield* fs.exists(path.join(canonicalRoot, "functions-one", "runtime", "functions")), - ).toBe(false); }).pipe(Effect.provide(NodeServices.layer)), ); }); diff --git a/packages/stack/src/host/CommandRunner.initialization.integration.test.ts b/packages/stack/src/host/CommandRunner.initialization.integration.test.ts index 86195a5867..f5420bf5ee 100644 --- a/packages/stack/src/host/CommandRunner.initialization.integration.test.ts +++ b/packages/stack/src/host/CommandRunner.initialization.integration.test.ts @@ -12,6 +12,7 @@ import { import { PreparationError } from "../preparation/Errors.ts"; import type { StackCredentials } from "../StackNamespace.ts"; import * as CommandRunner from "./CommandRunner.ts"; +import { noContainerClaims } from "../../tests/claims.ts"; const target = process.platform === "darwin" && process.arch === "arm64" @@ -62,6 +63,7 @@ const prepareAuthArtifact = Effect.fn(function* (cacheRoot: string, script: stri const makeRunner = Effect.fn(function* (root: string, cacheRoot: string) { const layer = CommandRunner.layer({ + claims: noContainerClaims, stackId: "initialization-command-test", root, cacheRoot, diff --git a/packages/stack/src/host/CommandRunner.native-cleanup.integration.test.ts b/packages/stack/src/host/CommandRunner.native-cleanup.integration.test.ts index 240d72a977..0a33d357c4 100644 --- a/packages/stack/src/host/CommandRunner.native-cleanup.integration.test.ts +++ b/packages/stack/src/host/CommandRunner.native-cleanup.integration.test.ts @@ -5,6 +5,7 @@ import { systemError } from "effect/PlatformError"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { postgres } from "../Commands.ts"; import * as CommandRunner from "./CommandRunner.ts"; +import { noContainerClaims } from "../../tests/claims.ts"; it.live.skipIf(process.platform === "win32")( "retries failed native workload cleanup when the stack runner is cleaned up", @@ -53,6 +54,7 @@ it.live.skipIf(process.platform === "win32")( }), ); const layer = CommandRunner.layer({ + claims: noContainerClaims, stackId: "native-cleanup-test", root, cacheRoot, diff --git a/packages/stack/src/host/CommandRunner.ts b/packages/stack/src/host/CommandRunner.ts index b75dc0696d..4074bf4b0b 100644 --- a/packages/stack/src/host/CommandRunner.ts +++ b/packages/stack/src/host/CommandRunner.ts @@ -21,7 +21,10 @@ import { postgresVersion, resolveArtifact, } from "../Artifacts.ts"; -import { makeContainerRuntime, type HostGateway } from "../runtime/Container.ts"; +import type * as Claims from "../namespace/Claims.ts"; +import * as Environment from "../namespace/Environment.ts"; +import { borrow } from "../namespace/Paths.ts"; +import { makeContainerRuntime, type EngineTarget, type HostGateway } from "../runtime/Container.ts"; import { spawnNativeProcess } from "../runtime/NativeProcess.ts"; import { awaitCommandOutput, type CommandOutputResult } from "../runtime/CommandOutput.ts"; import type { CommandInvocation as CommandInvocationType } from "../Commands.ts"; @@ -74,9 +77,13 @@ const makeCommandRunner = (options: { readonly project?: string; readonly root: string; readonly cacheRoot: string; - readonly runtime: "native" | "docker" | "podman"; + readonly runtime: "native" | "docker"; /** Shares one host-gateway probe with the host's other container runtimes. */ readonly hostGateway?: HostGateway; + /** The engine endpoint and identity the owner resolved once at startup; absent when native. */ + readonly engineTarget?: EngineTarget; + /** Journals each job container before it is created, for the namespace's reconcile loop. */ + readonly claims: Claims.ContainerClaims; }) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -86,14 +93,17 @@ const makeCommandRunner = (options: { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; if (!/^[a-zA-Z0-9_-]+$/u.test(options.stackId)) return yield* failure("Invalid stack identity"); const container = - options.runtime === "native" + options.engineTarget === undefined ? undefined : yield* makeContainerRuntime({ - engine: options.runtime, + target: options.engineTarget, root: options.root, imageMirrors: slimImageMirrors, ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), + claims: options.claims, }); + const borrowCallerPath = (candidate: string) => + borrow(fs, path, candidate, options.root, (_operation, cause) => failure(cause)); const jobsRoot = path.join(options.root, "jobs"); yield* fs .makeDirectory(jobsRoot, { recursive: true, mode: 0o700 }) @@ -165,6 +175,11 @@ const makeCommandRunner = (options: { postgresCommand.pgProve !== undefined ) return yield* failure("pgProve options require the pg_prove command"); + for (const mount of [ + ...(initialization?.mounts ?? []), + ...(postgresCommand?.pgProve?.mounts ?? []), + ]) + yield* borrowCallerPath(mount.source); const version = initialization?.version ?? postgresVersion(String(postgresCommand?.command.major)); const process = yield* Effect.gen(function* () { @@ -179,6 +194,11 @@ const makeCommandRunner = (options: { Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), Effect.provideService(HttpClient.HttpClient, http), ); + // The job's own scoped temp directory, removed with it, so the confined environment + // needs no cleanup of its own. + const environment = yield* Environment.confine(fs, path, directory).pipe( + Effect.mapError(failure), + ); const child = yield* spawnNativeProcess( { executable: path.join( @@ -188,6 +208,7 @@ const makeCommandRunner = (options: { ), args: postgresCommand?.args ?? initialization?.args ?? [], env: postgresCommand?.env ?? initialization?.env ?? {}, + environment, cwd: postgresCommand?.pgProve?.cwd ?? initialization?.cwd ?? @@ -308,7 +329,11 @@ export const layer = (options: { readonly project?: string; readonly root: string; readonly cacheRoot: string; - readonly runtime: "native" | "docker" | "podman"; + readonly runtime: "native" | "docker"; /** Shares one host-gateway probe with the host's other container runtimes. */ readonly hostGateway?: HostGateway; + /** The engine endpoint and identity the owner resolved once at startup; absent when native. */ + readonly engineTarget?: EngineTarget; + /** Journals each job container before it is created, for the namespace's reconcile loop. */ + readonly claims: Claims.ContainerClaims; }) => Layer.effect(Service, makeCommandRunner(options).pipe(Effect.map(Service.of))); diff --git a/packages/stack/src/internal/test-run-label.ts b/packages/stack/src/internal/test-run-label.ts index 8cdcdefb24..c0302f0839 100644 --- a/packages/stack/src/internal/test-run-label.ts +++ b/packages/stack/src/internal/test-run-label.ts @@ -1,6 +1,6 @@ import { Config, Effect, Option } from "effect"; -/** Environment variable a test run sets so stack-created Docker/Podman resources can be labelled. */ +/** Environment variable a test run sets so stack-created Docker resources can be labelled. */ export const testRunEnvVar = "SUPABASE_STACK_TEST_RUN"; /** Label key stamped on volumes and containers a test run creates. */ diff --git a/packages/stack/src/namespace/Claims.ts b/packages/stack/src/namespace/Claims.ts new file mode 100644 index 0000000000..494bcc0170 --- /dev/null +++ b/packages/stack/src/namespace/Claims.ts @@ -0,0 +1,177 @@ +import { Effect, FileSystem, Path, Schema, Semaphore } from "effect"; +import { namespaceError, type NamespaceError } from "./Capabilities.ts"; +import * as Publication from "./Publication.ts"; + +/** + * A resource the stack is about to create outside its own process lifetime: a container or a + * directory whose creation can be interrupted partway. Identity is exact, so reconciliation never + * guesses by name or label. `daemonId` records which engine daemon a container claim belongs to, + * so reconcile never removes (or mistakes) an identity recorded against a different daemon. + */ +export const ResourceClaim = Schema.Struct({ + kind: Schema.Literals(["container", "directory"]), + id: Schema.String, + daemonId: Schema.optionalKey(Schema.String), +}); +export interface ResourceClaim extends Schema.Schema.Type {} + +const ClaimsDocument = Schema.Struct({ claims: Schema.Array(ResourceClaim) }); + +/** The claims document file name, relative to a stack's own directory under the namespace root. */ +export const CLAIMS_FILE = "claims.json"; + +export interface Interface { + /** Journals `claim` before the resource it names is created; idempotent on a repeated identity. */ + readonly claim: (stackId: string, claim: ResourceClaim) => Effect.Effect; + /** Drops `claim` once its resource is removed, or once something else now owns its cleanup. */ + readonly unclaim: (stackId: string, claim: ResourceClaim) => Effect.Effect; + /** Every resource claim still recorded for `stackId`. */ + readonly readClaims: ( + stackId: string, + ) => Effect.Effect, NamespaceError>; +} + +export interface Options { + readonly root: string; + readonly platform?: NodeJS.Platform; +} + +const sameClaim = (left: ResourceClaim, right: ResourceClaim) => + left.kind === right.kind && left.id === right.id; + +export const make = ( + options: Options, +): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stackRoot = (id: string) => path.join(options.root, id); + const claimsPath = (id: string) => path.join(stackRoot(id), CLAIMS_FILE); + // Claims mutate only from within the single owner process that holds a stack's lease, so an + // in-process mutex per stack id is enough to serialize its read-modify-write cycle. + const locks = new Map(); + const lockFor = (id: string): Semaphore.Semaphore => { + const existing = locks.get(id); + if (existing !== undefined) return existing; + const created = Semaphore.makeUnsafe(1); + locks.set(id, created); + return created; + }; + + const readDocument = (id: string) => + fs.readFileString(claimsPath(id)).pipe( + Effect.mapError((cause) => namespaceError("read", cause)), + Effect.flatMap((text) => + Schema.decodeEffect(Schema.fromJsonString(ClaimsDocument))(text).pipe( + Effect.mapError((cause) => namespaceError("decode", cause)), + ), + ), + Effect.map((document) => document.claims), + ); + const read = (id: string): Effect.Effect, NamespaceError> => + fs.exists(claimsPath(id)).pipe( + Effect.mapError((cause) => namespaceError("read", cause)), + Effect.flatMap((exists) => (exists ? readDocument(id) : Effect.succeed([]))), + ); + + const write = ( + id: string, + claims: ReadonlyArray, + ): Effect.Effect => + fs.makeDirectory(stackRoot(id), { recursive: true, mode: 0o700 }).pipe( + Effect.mapError((cause) => namespaceError("write", cause)), + Effect.andThen( + Schema.encodeEffect(Schema.fromJsonString(ClaimsDocument))({ claims }).pipe( + Effect.mapError((cause) => namespaceError("encode", cause)), + ), + ), + Effect.flatMap((content) => + Publication.publish(fs, path, { + target: claimsPath(id), + content, + platform: options.platform, + }), + ), + ); + + const update = ( + id: string, + next: (current: ReadonlyArray) => ReadonlyArray, + ) => + lockFor(id).withPermits(1)( + read(id).pipe( + Effect.flatMap((current) => { + const updated = next(current); + // An unclaim (or a no-op) that still finds nothing recorded must not recreate the + // stack's directory: the stack may already be fully destroyed, with this call only a + // finalizer running after that, racing the directory it would otherwise resurrect. + if (current.length === 0 && updated.length === 0) return Effect.void; + return write(id, updated); + }), + ), + ); + + const claim = Effect.fn("Namespace.Claims.claim")((stackId: string, claimed: ResourceClaim) => + update(stackId, (current) => + current.some((entry) => sameClaim(entry, claimed)) ? current : [...current, claimed], + ), + ); + const unclaim = Effect.fn("Namespace.Claims.unclaim")( + (stackId: string, claimed: ResourceClaim) => + update(stackId, (current) => current.filter((entry) => !sameClaim(entry, claimed))), + ); + const readClaims = Effect.fn("Namespace.Claims.readClaims")((stackId: string) => read(stackId)); + + return { claim, unclaim, readClaims }; + }); + +/** A narrow capability a container runtime uses to journal exact container identities. */ +export interface ContainerClaims { + /** `daemonId`, when known, is recorded so reconcile never drops a claim seen from another daemon. */ + readonly claim: (id: string, daemonId?: string) => Effect.Effect; + readonly unclaim: (id: string) => Effect.Effect; +} + +/** A narrow capability for journaling a directory outside the owner's normal data tree. */ +export interface DirectoryClaims { + readonly claim: (directoryPath: string) => Effect.Effect; + readonly unclaim: (directoryPath: string) => Effect.Effect; +} + +/** Scopes the claims journal to one stack, exposing only its container and directory operations. */ +export const forStack = ( + state: Interface, + stackId: string, +): { readonly containers: ContainerClaims; readonly directories: DirectoryClaims } => ({ + containers: { + claim: (id, daemonId) => + state.claim(stackId, { + kind: "container", + id, + ...(daemonId === undefined ? {} : { daemonId }), + }), + unclaim: (id) => state.unclaim(stackId, { kind: "container", id }), + }, + directories: { + claim: (directoryPath) => state.claim(stackId, { kind: "directory", id: directoryPath }), + unclaim: (directoryPath) => state.unclaim(stackId, { kind: "directory", id: directoryPath }), + }, +}); + +/** + * Reads a stack's recorded claims and, for each one still present, asks `remove` to reconcile it: + * `"removed"` drops the claim, `"kept"` leaves it (for example a container claim recorded against + * a different daemon than the one reconcile is currently running against). A claim whose removal + * fails stays recorded, along with every claim after it, for the next acquisition to retry; it + * reports the first failure. + */ +export const reconcile = Effect.fn("Namespace.Claims.reconcile")(function* ( + claimsInterface: Pick, + stackId: string, + remove: (claim: ResourceClaim) => Effect.Effect<"removed" | "kept", E, R>, +): Effect.fn.Return { + for (const claim of yield* claimsInterface.readClaims(stackId)) { + const outcome = yield* remove(claim); + if (outcome === "removed") yield* claimsInterface.unclaim(stackId, claim); + } +}); diff --git a/packages/stack/src/namespace/Environment.integration.test.ts b/packages/stack/src/namespace/Environment.integration.test.ts new file mode 100644 index 0000000000..d1c349f6e4 --- /dev/null +++ b/packages/stack/src/namespace/Environment.integration.test.ts @@ -0,0 +1,52 @@ +import { NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Path, Schema } from "effect"; +import { confine, EnvironmentSymlinkError } from "./Environment.ts"; + +const run = (effect: Effect.Effect) => + Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); + +describe("native environment confinement", () => { + it.live("creates every confined directory under a fresh root", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "environment-confine-" }); + const environment = yield* confine(fs, path, root); + expect(yield* fs.exists(environment.values.XDG_CACHE_HOME)).toBe(true); + }), + ), + ); + + it.live( + "refuses to confine on restart when a reused component is a symlink planted by the step-down user, and creates nothing outside", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "environment-confine-symlink-", + }); + // First launch creates the confined layout normally. + yield* confine(fs, path, root); + const outside = yield* fs.makeTempDirectoryScoped({ + prefix: "environment-confine-outside-", + }); + const sentinel = path.join(outside, "sentinel"); + yield* fs.writeFileString(sentinel, "do not create here"); + // After handover, the step-down user that now owns `root` swaps `.cache` for a symlink + // to outside before the stack restarts. + const cache = path.join(root, ".cache"); + yield* fs.remove(cache, { recursive: true }); + yield* fs.symlink(outside, cache); + const failure = yield* confine(fs, path, root).pipe(Effect.flip); + if (!Schema.is(EnvironmentSymlinkError)(failure)) + return yield* Effect.die(`Expected an EnvironmentSymlinkError, got ${failure}`); + expect(failure.path).toBe(cache); + expect(yield* fs.readDirectory(outside)).toEqual(["sentinel"]); + }), + ), + ); +}); diff --git a/packages/stack/src/namespace/Environment.ts b/packages/stack/src/namespace/Environment.ts new file mode 100644 index 0000000000..32c8e601b4 --- /dev/null +++ b/packages/stack/src/namespace/Environment.ts @@ -0,0 +1,104 @@ +import { Effect, FileSystem, Path, Schema } from "effect"; +import type { PlatformError } from "effect/PlatformError"; +import { lstatPath } from "./drivers/FileSystem.ts"; + +/** + * Variables a native workload's environment is confined to. Every one of them is rooted under the + * owned directory {@link confine} is given, so a caller cannot redirect a workload's home, temp, + * cache, config, data, state or Deno directory outside it. + */ +const CONFINED_KEYS = [ + "HOME", + "TMPDIR", + "TMP", + "TEMP", + "XDG_CACHE_HOME", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + "XDG_STATE_HOME", + "DENO_DIR", +] as const; +type ConfinedKey = (typeof CONFINED_KEYS)[number]; + +/** The owned environment a native workload's spec must carry; constructed only by {@link confine}. */ +export interface NativeEnvironment { + readonly values: Readonly>; +} + +/** A caller tried to set an environment variable the namespace confines; it names every one. */ +export class EnvironmentOverrideError extends Schema.TaggedError()( + "Namespace.EnvironmentOverrideError", + { keys: Schema.Array(Schema.String) }, +) {} + +/** + * A reused environment component (the owned root or one nested under it) is a symlink or junction, + * which its current owner (a step-down user, for example) could have planted between launches. + */ +export class EnvironmentSymlinkError extends Schema.TaggedError()( + "Namespace.EnvironmentSymlinkError", + { path: Schema.String }, +) {} + +/** Every path component from `root` down to (and including) `target`, root first. */ +const componentsFrom = (path: Path.Path, root: string, target: string): ReadonlyArray => { + const relative = path.relative(root, target); + const segments = relative.split(path.sep).filter((segment) => segment.length > 0); + const components: Array = [root]; + let current = root; + for (const segment of segments) { + current = path.join(current, segment); + components.push(current); + } + return components; +}; + +/** + * Builds the confined environment for a native workload rooted at `root`: `root` itself is HOME, + * with a temp, cache, config, data, state and Deno directory nested under it. On reuse, a prior + * launch's confined directories could have been replaced with a symlink or junction by their + * current owner; every component from `root` down is rejected if so, before anything is created + * through it. Creates every directory so the workload finds them ready on first use. + */ +export const confine = Effect.fn("Namespace.Environment.confine")(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + root: string, +): Effect.fn.Return { + const home = root; + const tmp = path.join(home, "tmp"); + const values: Record = { + HOME: home, + TMPDIR: tmp, + TMP: tmp, + TEMP: tmp, + XDG_CACHE_HOME: path.join(home, ".cache"), + XDG_CONFIG_HOME: path.join(home, ".config"), + XDG_DATA_HOME: path.join(home, ".local", "share"), + XDG_STATE_HOME: path.join(home, ".local", "state"), + DENO_DIR: path.join(home, ".cache", "deno"), + }; + const directories = new Set(Object.values(values)); + const components = new Set([root]); + for (const directory of directories) + for (const component of componentsFrom(path, root, directory)) components.add(component); + for (const component of components) + if ((yield* lstatPath(component))?.type === "SymbolicLink") + return yield* new EnvironmentSymlinkError({ path: component }); + for (const directory of directories) + yield* fs.makeDirectory(directory, { recursive: true, mode: 0o700 }); + return { values }; +}); + +/** + * Merges `environment`'s confined values into `callerEnv`, failing instead of letting `callerEnv` + * override any of them. + */ +export const apply = ( + environment: NativeEnvironment, + callerEnv: Readonly> = {}, +): Effect.Effect, EnvironmentOverrideError> => { + const overridden = CONFINED_KEYS.filter((key) => key in callerEnv); + if (overridden.length > 0) return Effect.fail(new EnvironmentOverrideError({ keys: overridden })); + return Effect.succeed({ ...callerEnv, ...environment.values }); +}; diff --git a/packages/stack/src/namespace/Namespace.e2e.test.ts b/packages/stack/src/namespace/Namespace.e2e.test.ts index 1b16f1d8d0..04aceb258b 100644 --- a/packages/stack/src/namespace/Namespace.e2e.test.ts +++ b/packages/stack/src/namespace/Namespace.e2e.test.ts @@ -22,8 +22,9 @@ import { shutdownHost, waitForOwnerExit, } from "../HostProcess.ts"; +import { create, open } from "../effect.ts"; import * as StackNamespace from "../StackNamespace.ts"; -import { watchLeaseRelease } from "../../tests/owner.ts"; +import { captureOwnerPid, watchLeaseRelease } from "../../tests/owner.ts"; class PublishBarrierError extends Data.TaggedError("PublishBarrierError")<{ readonly message: string; @@ -316,3 +317,100 @@ it.live( ).pipe(Effect.provide(testLayer)), 20_000, ); + +const docker = Effect.fn("Namespace.e2e.docker")((args: ReadonlyArray) => + Effect.scoped( + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + ChildProcess.make("docker", args, { stdin: "ignore", stdout: "pipe", stderr: "pipe" }), + ); + const [stdout, stderr, code] = yield* Effect.all( + [ + Stream.mkString(Stream.decodeText(child.stdout)), + Stream.mkString(Stream.decodeText(child.stderr)), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + if (Number(code) !== 0) + return yield* Effect.die(`docker ${args.join(" ")} failed: ${stderr}`); + return stdout.trim(); + }), + ), +); + +it.live.skipIf(process.platform === "win32")( + "removes exactly the containers a SIGKILLed owner's claims recorded, with no manual cleanup", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-e2e-container-" }); + const stateRoot = `${root}/state`; + const cacheRoot = `${root}/cache`; + const stack = yield* create({ + projectRoot: root, + stateRoot, + cacheRoot, + runtime: "docker", + startOwner: true, + }); + yield* Effect.addFinalizer(() => stack.destroy.pipe(Effect.ignore)); + const mail = yield* stack.services.create({ service: "mail", config: {} }); + yield* mail.start; + yield* mail.ready; + + const containerIds = (yield* docker([ + "ps", + "--all", + "--quiet", + "--no-trunc", + "--filter", + `label=com.supabase.stack=${stack.id}`, + "--filter", + `label=com.supabase.instance=${mail.id}`, + ])) + .split("\n") + .filter((id) => id.length > 0); + expect(containerIds).toHaveLength(1); + const [containerId] = containerIds; + + const ownerPid = yield* captureOwnerPid({ stateRoot, cacheRoot }, stack.id); + const released = yield* watchLeaseRelease(stateRoot, stack.id); + yield* Effect.sync(() => process.kill(ownerPid, "SIGKILL")); + yield* released; + + // Killing the owner leaves the container running; nothing in this test removes it. + expect( + yield* docker(["inspect", "--format", "{{.State.Status}}", String(containerId)]), + ).not.toBe(""); + + const reopened = yield* open({ + id: stack.id, + stateRoot, + cacheRoot, + startOwner: true, + }); + yield* Effect.addFinalizer(() => reopened.destroy.pipe(Effect.ignore)); + + const remaining = yield* docker([ + "ps", + "--all", + "--quiet", + "--no-trunc", + "--filter", + `label=com.supabase.stack=${stack.id}`, + ]); + expect(remaining).toBe(""); + const inspectExit = yield* docker([ + "inspect", + "--format", + "{{.State.Status}}", + String(containerId), + ]).pipe(Effect.exit); + expect(inspectExit._tag).toBe("Failure"); + }), + ).pipe(Effect.provide(testLayer)), + 120_000, +); diff --git a/packages/stack/src/namespace/Paths.integration.test.ts b/packages/stack/src/namespace/Paths.integration.test.ts new file mode 100644 index 0000000000..5e5ed5cf61 --- /dev/null +++ b/packages/stack/src/namespace/Paths.integration.test.ts @@ -0,0 +1,223 @@ +import { NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Data, Effect, FileSystem, Path } from "effect"; +import { borrow, destroyOwnedRoot } from "./Paths.ts"; + +class TestPathsError extends Data.TaggedError("TestPathsError")<{ + readonly operation: string; + readonly cause: unknown; +}> {} + +const onError = (operation: string, cause: unknown) => new TestPathsError({ operation, cause }); + +const run = (effect: Effect.Effect) => + Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); + +describe("ownership is by location", () => { + it.live("rejects a borrowed path that resolves inside the stack's data root", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dataRoot = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-reject-" }); + const ownedRoot = path.join(dataRoot, "instance"); + yield* fs.makeDirectory(ownedRoot, { recursive: true }); + const inside = path.join(ownedRoot, "config.yaml"); + yield* fs.writeFileString(inside, "borrowed"); + const failure = yield* borrow(fs, path, inside, dataRoot, onError).pipe(Effect.flip); + expect(failure.operation).toBe("configure"); + }), + ), + ); + + it.live("rejects a borrowed path equal to the stack's data root", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dataRoot = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-equal-" }); + const failure = yield* borrow(fs, path, dataRoot, dataRoot, onError).pipe(Effect.flip); + expect(failure.operation).toBe("configure"); + }), + ), + ); + + it.live("rejects a borrowed path inside a sibling instance under the same data root", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dataRoot = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-sibling-" }); + const siblingInstance = path.join(dataRoot, "mail-instance"); + yield* fs.makeDirectory(siblingInstance, { recursive: true }); + const inside = path.join(siblingInstance, "pipeline.yaml"); + yield* fs.writeFileString(inside, "borrowed"); + const failure = yield* borrow(fs, path, inside, dataRoot, onError).pipe(Effect.flip); + expect(failure.operation).toBe("configure"); + }), + ), + ); + + it.live("accepts a borrowed path outside the stack's data root", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dataRoot = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-accept-" }); + const outside = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-caller-" }).pipe( + Effect.flatMap((root) => { + const file = path.join(root, "caller-config.yaml"); + return fs.writeFileString(file, "borrowed").pipe(Effect.as(file)); + }), + ); + const borrowed = yield* borrow(fs, path, outside, dataRoot, onError); + expect(borrowed.path).toBe(outside); + }), + ), + ); + + it.live("accepts a borrowed path that does not exist yet, under a real ancestor", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dataRoot = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-pending-" }); + const outsideRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "owned-root-pending-out-", + }); + const notYetCreated = path.join(outsideRoot, "not-created-yet.yaml"); + yield* borrow(fs, path, notYetCreated, dataRoot, onError); + }), + ), + ); + + it.live("recursively removes a real owned root after running removeData", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const parentRoot = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-destroy-" }); + const ownedRoot = path.join(parentRoot, "instance"); + yield* fs.makeDirectory(path.join(ownedRoot, "nested"), { recursive: true }); + yield* fs.writeFileString(path.join(ownedRoot, "nested", "file"), "data"); + let removeDataRan = false; + yield* destroyOwnedRoot( + fs, + path, + ownedRoot, + parentRoot, + Effect.sync(() => { + removeDataRan = true; + }), + onError, + ); + expect(removeDataRan).toBe(true); + expect(yield* fs.exists(ownedRoot)).toBe(false); + }), + ), + ); + + it.live("runs removeData even when the owned root was never created", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const parentRoot = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-absent-" }); + const ownedRoot = path.join(parentRoot, "instance"); + let removeDataRan = false; + yield* destroyOwnedRoot( + fs, + path, + ownedRoot, + parentRoot, + Effect.sync(() => { + removeDataRan = true; + }), + onError, + ); + expect(removeDataRan).toBe(true); + }), + ), + ); + + it.live("keeps listed entries after removeData, removing the rest", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const parentRoot = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-keep-" }); + const ownedRoot = path.join(parentRoot, "instance"); + yield* fs.makeDirectory(ownedRoot, { recursive: true }); + yield* fs.writeFileString(path.join(ownedRoot, "snapshots"), "keep-me"); + yield* fs.writeFileString(path.join(ownedRoot, "data"), "remove-me"); + yield* destroyOwnedRoot(fs, path, ownedRoot, parentRoot, Effect.void, onError, [ + "snapshots", + ]); + expect(yield* fs.readDirectory(ownedRoot)).toEqual(["snapshots"]); + }), + ), + ); + + it.live( + "refuses to destroy a root that is a symlink to an outside directory, which survives", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const parentRoot = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-symlink-" }); + const outside = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-outside-" }); + const sentinel = path.join(outside, "sentinel"); + yield* fs.writeFileString(sentinel, "do not remove me"); + const ownedRoot = path.join(parentRoot, "instance"); + yield* fs.symlink(outside, ownedRoot); + // Destructive: if confirmRealOwnedDirectory ran after this (or not at all), the sentinel + // outside the owned root would actually be deleted, not just silently skipped. + const failure = yield* destroyOwnedRoot( + fs, + path, + ownedRoot, + parentRoot, + fs + .remove(sentinel, { force: true }) + .pipe(Effect.mapError((cause) => onError("destroy", cause))), + onError, + ).pipe(Effect.flip); + expect(failure.operation).toBe("destroy"); + expect(yield* fs.exists(sentinel)).toBe(true); + }), + ), + ); + + it.live("refuses to destroy a symlinked root even when keep entries are requested", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const parentRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "owned-root-symlink-keep-", + }); + const outside = yield* fs.makeTempDirectoryScoped({ + prefix: "owned-root-outside-keep-", + }); + const sentinel = path.join(outside, "sentinel"); + yield* fs.writeFileString(sentinel, "do not remove me"); + const ownedRoot = path.join(parentRoot, "instance"); + yield* fs.symlink(outside, ownedRoot); + const failure = yield* destroyOwnedRoot( + fs, + path, + ownedRoot, + parentRoot, + fs + .remove(sentinel, { force: true }) + .pipe(Effect.mapError((cause) => onError("destroy", cause))), + onError, + ["sentinel"], + ).pipe(Effect.flip); + expect(failure.operation).toBe("destroy"); + expect(yield* fs.exists(sentinel)).toBe(true); + }), + ), + ); +}); diff --git a/packages/stack/src/namespace/Paths.ts b/packages/stack/src/namespace/Paths.ts new file mode 100644 index 0000000000..2ca179f325 --- /dev/null +++ b/packages/stack/src/namespace/Paths.ts @@ -0,0 +1,196 @@ +import { Effect, FileSystem, Path } from "effect"; +import { lstatPath } from "./drivers/FileSystem.ts"; + +const BorrowedPathBrand: unique symbol = Symbol("Namespace.BorrowedPath"); + +/** + * A caller-supplied path the namespace only reads or mounts, never deletes. The brand makes + * {@link borrow} the only way to construct one: a service that needs a caller path in `prepare`, + * `mounts`, `env`, or `args` must hold a `BorrowedPath`, so a path that was never validated against + * the stack's data root cannot be used in its place. + */ +export interface BorrowedPath { + readonly path: string; + readonly [BorrowedPathBrand]: true; +} + +/** + * Resolves `target`'s real path, walking up to its nearest existing ancestor and rejoining the + * remaining segments when it (or an ancestor) does not exist yet, so a symlinked ancestor cannot + * smuggle a not-yet-created path inside an owned root. + */ +const resolveReal = ( + fs: FileSystem.FileSystem, + path: Path.Path, + target: string, + onError: (operation: string, cause: unknown) => E, +): Effect.Effect => + Effect.gen(function* () { + const exists = yield* fs + .exists(target) + .pipe(Effect.mapError((cause) => onError("configure", cause))); + if (exists) + return yield* fs + .realPath(target) + .pipe(Effect.mapError((cause) => onError("configure", cause))); + const parent = path.dirname(target); + if (parent === target) return path.resolve(target); + const realParent = yield* resolveReal(fs, path, parent, onError); + return path.join(realParent, path.basename(target)); + }); + +/** + * The only way to construct a {@link BorrowedPath}. Ownership is by location: everything under the + * stack's data root belongs to the stack, so a borrowed (read-only) caller path must resolve + * neither to it nor inside it — checked by real path, equality included, after resolving through + * any ancestor symlink. Call before preparation or launch uses `candidate`. + */ +export const borrow = Effect.fn("Namespace.Paths.borrow")( + ( + fs: FileSystem.FileSystem, + path: Path.Path, + candidate: string, + dataRoot: string, + onError: (operation: string, cause: unknown) => E, + ): Effect.Effect => + Effect.gen(function* () { + const [realCandidate, realDataRoot] = yield* Effect.all([ + resolveReal(fs, path, candidate, onError), + resolveReal(fs, path, dataRoot, onError), + ]); + if (realCandidate === realDataRoot || realCandidate.startsWith(`${realDataRoot}${path.sep}`)) + return yield* Effect.fail( + onError("configure", `${candidate} resolves inside the owned data root ${dataRoot}`), + ); + return { path: candidate, [BorrowedPathBrand]: true }; + }), +); + +/** Reports whether `candidate`'s real path is inside, or equal to, the real path of any of `roots`. */ +export const isWithinOwnedRoots = Effect.fn("Namespace.Paths.isWithinOwnedRoots")( + ( + fs: FileSystem.FileSystem, + path: Path.Path, + candidate: string, + roots: ReadonlyArray, + onError: (operation: string, cause: unknown) => E, + ): Effect.Effect => + Effect.gen(function* () { + const realCandidate = yield* resolveReal(fs, path, candidate, onError); + for (const root of roots) { + const realRoot = yield* resolveReal(fs, path, root, onError); + if (realCandidate === realRoot || realCandidate.startsWith(`${realRoot}${path.sep}`)) + return true; + } + return false; + }), +); + +/** The container mount target for an instance-scoped directory owned on the host. */ +export const containerInstancePath = "/instance"; + +/** Owned per-launch environment-file directory name under a stack's data root. */ +export const CONTAINER_ENV_DIRNAME = ".container-env"; + +/** + * Confirms `root` is a real directory (not a symlink or junction) whose real path resolves inside + * `parentRoot`'s real path. {@link destroyOwnedRoot} runs this before any destructive step, + * including the `keep` branch, so a symlink planted at `root` (or replacing it between launches) + * is refused rather than traversed or removed. + */ +const confirmRealOwnedDirectory = ( + fs: FileSystem.FileSystem, + path: Path.Path, + root: string, + parentRoot: string, + onError: (operation: string, cause: unknown) => E, +): Effect.Effect => + Effect.gen(function* () { + const info = yield* lstatPath(root).pipe(Effect.mapError((cause) => onError("destroy", cause))); + if (info?.type === "SymbolicLink") + return yield* Effect.fail(onError("destroy", `${root} is a symlink, not an owned directory`)); + if (info === undefined || info.type !== "Directory") + return yield* Effect.fail(onError("destroy", `${root} is not a directory`)); + const realRoot = yield* fs + .realPath(root) + .pipe(Effect.mapError((cause) => onError("destroy", cause))); + const realParent = yield* fs + .realPath(parentRoot) + .pipe(Effect.mapError((cause) => onError("destroy", cause))); + if (realRoot !== realParent && !realRoot.startsWith(`${realParent}${path.sep}`)) + return yield* Effect.fail(onError("destroy", `${root} resolves outside ${parentRoot}`)); + }); + +/** + * Destroys the owned root `root` (a child of `parentRoot`), confirming it first in every branch, + * including `keep`, and before `removeData` runs. `removeData` always runs, even when `root` was + * never created on disk, since a recipe's data may live entirely outside the host filesystem (an + * engine volume, for example). One race is accepted and left unguarded: an ancestor directory + * replaced between the confirmation below and the removal that follows it — a stack's own owner is + * the only writer of its roots. + */ +export const destroyOwnedRoot = Effect.fn("Namespace.Paths.destroyOwnedRoot")( + ( + fs: FileSystem.FileSystem, + path: Path.Path, + root: string, + parentRoot: string, + removeData: Effect.Effect, + onError: (operation: string, cause: unknown) => E, + /** Root entries kept after removeData; when empty the root itself is removed. */ + keep: ReadonlyArray = [], + ): Effect.Effect => + Effect.gen(function* () { + const exists = yield* fs + .exists(root) + .pipe(Effect.mapError((cause) => onError("destroy", cause))); + if (exists) yield* confirmRealOwnedDirectory(fs, path, root, parentRoot, onError); + yield* removeData; + if (!exists) return; + if (keep.length === 0) { + yield* fs + .remove(root, { recursive: true, force: true }) + .pipe(Effect.mapError((cause) => onError("destroy", cause))); + return; + } + const names = yield* fs + .readDirectory(root) + .pipe(Effect.mapError((cause) => onError("destroy", cause))); + for (const name of names) + if (!keep.includes(name)) + yield* fs + .remove(path.join(root, name), { recursive: true, force: true }) + .pipe(Effect.mapError((cause) => onError("destroy", cause))); + }), +); + +/** + * Writes `content` to `target` atomically (full content staged beside it, then renamed into + * place), for a file the namespace owns inside a directory a caller may also write into. + */ +export const writeOwnedFile = Effect.fn("Namespace.Paths.writeOwnedFile")( + ( + fs: FileSystem.FileSystem, + path: Path.Path, + target: string, + content: string, + onError: (operation: string, cause: unknown) => E, + ): Effect.Effect => + Effect.gen(function* () { + const directory = path.dirname(target); + yield* fs.makeDirectory(directory, { recursive: true, mode: 0o700 }); + yield* Effect.acquireUseRelease( + fs.makeTempDirectory({ directory, prefix: `.${path.basename(target)}-` }), + (temporaryDirectory) => + Effect.gen(function* () { + const temporary = path.join(temporaryDirectory, path.basename(target)); + yield* fs.writeFileString(temporary, content, { mode: 0o644 }); + yield* fs.rename(temporary, target); + }), + (temporaryDirectory) => + fs + .remove(temporaryDirectory, { recursive: true, force: true }) + .pipe(Effect.catchTag("PlatformError", () => Effect.void)), + ); + }).pipe(Effect.mapError((cause) => onError("prepare", cause))), +); diff --git a/packages/stack/src/namespace/Registry.ts b/packages/stack/src/namespace/Registry.ts index 710bc8f182..356d5ea113 100644 --- a/packages/stack/src/namespace/Registry.ts +++ b/packages/stack/src/namespace/Registry.ts @@ -7,6 +7,8 @@ import { transientRetrySchedule, type NamespaceError, } from "./Capabilities.ts"; +import { CLAIMS_FILE } from "./Claims.ts"; +import { CONTAINER_ENV_DIRNAME } from "./Paths.ts"; import * as Publication from "./Publication.ts"; import { removeEmptyDirectory } from "./drivers/FileSystem.ts"; import { acquireLock, isBusy, takeLock } from "./drivers/Sqlite.ts"; @@ -73,7 +75,7 @@ export const SavedStack = Schema.Struct({ branchContext: Schema.String, stackName: Schema.String, }), - runtime: Schema.Literals(["native", "docker", "podman"]), + runtime: Schema.Literals(["native", "docker"]), instances: Schema.Array(SavedInstance).check( Schema.makeFilter((instances) => new Set(instances.map(({ id }) => id)).size === instances.length @@ -214,10 +216,19 @@ export const make = ( statePath(id), path.join(stackRoot(id), "owner.json"), path.join(stackRoot(id), "owner.log"), + path.join(stackRoot(id), CLAIMS_FILE), ]) yield* fs .remove(file, { force: true }) .pipe(Effect.mapError((cause) => namespaceError("remove", cause))); + // Namespace-owned scratch space, never user data, so a forced recursive removal here (and + // only here) doesn't mask a service that failed to clean up its own owned root. + yield* fs + .remove(path.join(stackRoot(id), "data", CONTAINER_ENV_DIRNAME), { + recursive: true, + force: true, + }) + .pipe(Effect.mapError((cause) => namespaceError("remove", cause))); yield* removeEmptyDirectory(path.join(stackRoot(id), "data")); yield* removeEmptyDirectory(stackRoot(id)); }); diff --git a/packages/stack/src/namespace/drivers/FileSystem.ts b/packages/stack/src/namespace/drivers/FileSystem.ts index 6bd2be7172..962c0185e9 100644 --- a/packages/stack/src/namespace/drivers/FileSystem.ts +++ b/packages/stack/src/namespace/drivers/FileSystem.ts @@ -1,9 +1,51 @@ import { Effect, FileSystem, Predicate } from "effect"; -import type { PlatformError } from "effect/PlatformError"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no non-recursive directory removal operation. -import { rmdir } from "node:fs/promises"; +import { systemError, type PlatformError } from "effect/PlatformError"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has neither a no-follow stat nor a non-recursive directory removal. +import { lstat, rmdir } from "node:fs/promises"; import { namespaceError, type NamespaceError } from "../Capabilities.ts"; +export interface LstatInfo { + readonly type: "Directory" | "SymbolicLink" | "File" | "Other"; + readonly uid: number; + readonly mode: number; +} + +const toLstatInfo = (stats: Awaited>): LstatInfo => ({ + type: stats.isSymbolicLink() + ? "SymbolicLink" + : stats.isDirectory() + ? "Directory" + : stats.isFile() + ? "File" + : "Other", + uid: Number(stats.uid), + mode: Number(stats.mode), +}); + +const toLstatError = (path: string, cause: unknown): PlatformError => + systemError({ + _tag: Predicate.hasProperty(cause, "code") && cause.code === "ENOENT" ? "NotFound" : "Unknown", + module: "FileSystem", + method: "lstat", + pathOrDescriptor: path, + cause, + }); + +/** + * No-follow metadata for `path`: a single `lstat` syscall that never resolves a symlink, closing + * the time-of-check gap between a probe and a later symlink-following stat. Resolves to `undefined` + * when `path` has no entry at all. + */ +export const lstatPath = (path: string): Effect.Effect => + Effect.tryPromise({ try: () => lstat(path), catch: (cause) => toLstatError(path, cause) }).pipe( + Effect.map(toLstatInfo), + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + // oxlint-disable-next-line effecttsgo/effect-succeed-with-void -- the success type is `undefined`, not `void`. + () => Effect.succeed(undefined), + ), + ); + /** Removes a directory only if it is already empty; any other failure propagates. */ export const removeEmptyDirectory = (directory: string): Effect.Effect => Effect.tryPromise({ diff --git a/packages/stack/src/runtime/Container.integration.test.ts b/packages/stack/src/runtime/Container.integration.test.ts index 413f859be4..9e16d07d1e 100644 --- a/packages/stack/src/runtime/Container.integration.test.ts +++ b/packages/stack/src/runtime/Container.integration.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "@effect/vitest"; import { Cause, ConfigProvider, + Context, Crypto, Data, Deferred, @@ -13,6 +14,7 @@ import { Layer, Option, Path, + Redacted, Ref, Schema, Sink, @@ -27,8 +29,15 @@ import { DOCKER_HOST_ALIAS, makeContainerRuntime, makeHostGateway, + resolveEngineTarget, type ContainerProcess, + type EngineTarget, } from "./Container.ts"; +import * as Owner from "../Owner.ts"; +import * as StackNamespace from "../StackNamespace.ts"; +import { makeDockerDatabaseStorage } from "../storage/DockerDatabaseStorage.ts"; +import { reconcileContainerPassword } from "../services/Database.ts"; +import { noContainerClaims } from "../../tests/claims.ts"; const image = await Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -37,6 +46,9 @@ const image = await Effect.gen(function* () { const version = yield* fs.readFileString(file); return `oven/bun:${version.trim()}-slim`; }).pipe(Effect.provide(NodeServices.layer), Effect.runPromise); +// An unpinned target: these tests exercise the container runtime, not endpoint pinning, so an +// empty argv prefix leaves every command exactly as it was before pinning existed. +const dockerTarget: EngineTarget = { engine: "docker", argv: [], daemonId: "test-daemon-id" }; const ipv4 = /^(?:\d{1,3}\.){3}\d{1,3}$/u; const stoppableIdleScript = "process.on('SIGTERM', () => process.exit(0)); setInterval(() => {}, 1000)"; @@ -50,7 +62,11 @@ describe("container process adapter", () => { it.live("recognizes a cached pinned image without pulling", () => Effect.gen(function* () { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const repositoryDigest = yield* repoDigest(delegate, image); const digest = repositoryDigest.slice(repositoryDigest.indexOf("@") + 1); @@ -58,7 +74,11 @@ describe("container process adapter", () => { const pinnedImage = `${image}@${digest}`; const pullAttempted = yield* Ref.make(false); const spawner = makePullFailureSpawner(delegate, pullAttempted); - yield* makeContainerRuntime({ engine: "docker", root: "." }).pipe( + yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }).pipe( Effect.flatMap((runtime) => runtime.prepare(pinnedImage)), Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), ); @@ -73,7 +93,11 @@ describe("container process adapter", () => { const token = yield* crypto.randomUUIDv4; const pullAttempted = yield* Ref.make(false); const spawner = makePullFailureSpawner(delegate, pullAttempted); - const result = yield* makeContainerRuntime({ engine: "docker", root: "." }).pipe( + const result = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }).pipe( Effect.flatMap((runtime) => runtime.prepare(`supabase-prepare-regression:${token}`).pipe(Effect.exit), ), @@ -89,7 +113,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const id = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -123,7 +151,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const id = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const process = yield* runtime.launchCommand({ image, @@ -160,7 +192,8 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - engine: "docker", + claims: noContainerClaims, + target: dockerTarget, root: ".", hostGateway: yield* makeHostGateway, }); @@ -197,7 +230,12 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const hostGateway = yield* makeHostGateway; - const makeRuntime = makeContainerRuntime({ engine: "docker", root: ".", hostGateway }).pipe( + const makeRuntime = makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + hostGateway, + }).pipe( Effect.provideService( ChildProcessSpawner.ChildProcessSpawner, makeHostGatewayProbeSpawner(delegate, probes, () => undefined), @@ -239,13 +277,15 @@ describe("container process adapter", () => { const hostGateway = yield* makeHostGateway; const spawner = makeHostGatewayProbeSpawner(delegate, probes, () => undefined, release); const database = yield* makeContainerRuntime({ - engine: "docker", + claims: noContainerClaims, + target: dockerTarget, root: ".", hostGateway, awaitHostGateway: false, }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)); const service = yield* makeContainerRuntime({ - engine: "docker", + claims: noContainerClaims, + target: dockerTarget, root: ".", hostGateway, }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)); @@ -280,7 +320,8 @@ describe("container process adapter", () => { Effect.gen(function* () { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const runtime = yield* makeContainerRuntime({ - engine: "docker", + claims: noContainerClaims, + target: dockerTarget, root: ".", hostGateway: yield* makeHostGateway, }).pipe( @@ -316,7 +357,8 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - engine: "docker", + claims: noContainerClaims, + target: dockerTarget, root: ".", hostGateway: yield* makeHostGateway, }).pipe( @@ -354,7 +396,8 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - engine: "docker", + claims: noContainerClaims, + target: dockerTarget, root: ".", hostGateway: yield* makeHostGateway, }).pipe( @@ -393,14 +436,15 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - engine: "docker", + claims: noContainerClaims, + target: dockerTarget, root: ".", hostGateway: yield* makeHostGateway, }).pipe( Effect.provideService( ChildProcessSpawner.ChildProcessSpawner, makeHostGatewayProbeSpawner(delegate, probes, (attempt) => - attempt === 1 ? hostGatewayRejectionScript : podmanHostsScript, + attempt === 1 ? hostGatewayRejectionScript : engineHostsScript, ), ), ); @@ -425,7 +469,8 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - engine: "docker", + claims: noContainerClaims, + target: dockerTarget, root: ".", hostGateway: yield* makeHostGateway, }).pipe( @@ -435,7 +480,7 @@ describe("container process adapter", () => { attempt === 2 ? `console.error("daemon busy"); process.exit(1)` : attempt === 4 - ? podmanHostsScript + ? engineHostsScript : hostGatewayRejectionScript, ), ), @@ -460,7 +505,8 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - engine: "docker", + claims: noContainerClaims, + target: dockerTarget, root: ".", hostGateway: yield* makeHostGateway, }).pipe( @@ -485,7 +531,9 @@ describe("container process adapter", () => { }), ).pipe(Effect.flip); expect(failure._tag).toBe("ContainerError"); - expect(failure.message).toContain("upgrade Podman or use --runtime podman"); + expect(failure.message).toContain( + "rejects host-gateway in --add-host and maps no IPv4 address", + ); } expect(yield* Ref.get(probes)).toBe(2); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), @@ -497,7 +545,8 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const database = yield* makeContainerRuntime({ - engine: "docker", + claims: noContainerClaims, + target: dockerTarget, root: ".", hostGateway: yield* makeHostGateway, awaitHostGateway: false, @@ -507,7 +556,7 @@ describe("container process adapter", () => { makeHostGatewayProbeSpawner( makeHostGatewayRejectingCreateSpawner(delegate), probes, - (attempt) => (attempt === 1 ? hostGatewayRejectionScript : podmanHostsScript), + (attempt) => (attempt === 1 ? hostGatewayRejectionScript : engineHostsScript), ), ), ); @@ -529,7 +578,11 @@ describe("container process adapter", () => { it.live("names and labels a service container for compose-style grouping", () => Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -553,7 +606,11 @@ describe("container process adapter", () => { it.live("marks a one-shot container's name and compose labels as a task", () => Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const process = yield* runtime.launchCommand({ image, @@ -576,7 +633,11 @@ describe("container process adapter", () => { it.live("labels a created container with the configured test run", () => Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const crypto = yield* Crypto.Crypto; const testRunId = `container-test-run-${(yield* crypto.randomUUIDv4).slice(0, 8)}`; @@ -606,7 +667,11 @@ describe("container process adapter", () => { () => Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const launch = (instanceId: string, marker: string) => runtime.launch({ @@ -642,7 +707,11 @@ describe("container process adapter", () => { it.live("stops a container with its configured stop signal", () => Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -668,7 +737,11 @@ describe("container process adapter", () => { it.live("waits until a discarded container stops before returning", () => Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -695,7 +768,11 @@ describe("container process adapter", () => { it.live("returns cleanup authority when container start fails", () => Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const result = yield* runtime .launch({ @@ -723,7 +800,11 @@ describe("container process adapter", () => { it.live("keeps exit observation shared after a caller cancels its wait", () => Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -751,7 +832,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const released = yield* Deferred.make(); - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }).pipe( + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }).pipe( Effect.provideService( ChildProcessSpawner.ChildProcessSpawner, makeLogFollowerSpawner( @@ -789,7 +874,11 @@ describe("container process adapter", () => { const released = yield* Deferred.make(); yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }).pipe( + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }).pipe( Effect.provideService( ChildProcessSpawner.ChildProcessSpawner, makeLogFollowerSpawner(delegate, released, "setInterval(() => {}, 1000)"), @@ -819,7 +908,11 @@ describe("container process adapter", () => { yield* Effect.ensuring( Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -851,7 +944,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const result = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -888,7 +985,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const result = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -930,7 +1031,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const result = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -976,7 +1081,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const result = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -1016,7 +1125,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const result = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -1063,7 +1176,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const result = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -1107,7 +1224,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const result = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -1160,7 +1281,11 @@ describe("container process adapter", () => { yield* Effect.ensuring( Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const result = yield* runtime .launch({ @@ -1201,7 +1326,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const result = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); return yield* runtime .launch({ @@ -1241,7 +1370,11 @@ describe("container process adapter", () => { const spawner = makePendingCreateSpawner(delegate, started, release, present); const result = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); yield* runtime.prepare(image); const launch = yield* runtime .launch({ @@ -1270,9 +1403,11 @@ describe("container process adapter", () => { it.effect("bounds a hung docker create and removes the container it may still create", () => Effect.gen(function* () { const engine = yield* makeHangingCreateSpawner(); - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }).pipe( - Effect.provide(engine.layer), - ); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }).pipe(Effect.provide(engine.layer)); const launch = yield* Effect.scoped( runtime.launch({ image, stackId: "i".repeat(64), instanceId: "hung-create", env: {} }), ).pipe(Effect.provide(engine.layer), Effect.exit, Effect.forkChild); @@ -1294,9 +1429,11 @@ describe("container process adapter", () => { it.effect("removes the container a hung docker create may still create when interrupted", () => Effect.gen(function* () { const engine = yield* makeHangingCreateSpawner(); - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }).pipe( - Effect.provide(engine.layer), - ); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }).pipe(Effect.provide(engine.layer)); const launch = yield* Effect.scoped( runtime.launch({ image, stackId: "i".repeat(64), instanceId: "interrupted", env: {} }), ).pipe(Effect.provide(engine.layer), Effect.forkChild); @@ -1307,6 +1444,195 @@ describe("container process adapter", () => { expect(engine.commands).toContainEqual(["rm", "--force", engine.createdName()]); }).pipe(Effect.provide(NodeServices.layer)), ); + + it.live( + "resolveEngineTarget pins the context resolved at startup by name, unaffected by a later context switch", + () => + Effect.gen(function* () { + const contextCalls: Array> = []; + let activeContext = "alpha"; + const handle = (exitCode: number, stdout = "") => + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(0), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(exitCode)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + stdout: Stream.succeed(new TextEncoder().encode(stdout)), + stderr: Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + const spawner = ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command) || command.command !== "docker") + return Effect.die("Unexpected command"); + if (command.args[0] === "context" && command.args[1] === "show") { + contextCalls.push(command.args); + return Effect.succeed(handle(0, `${activeContext}\n`)); + } + // Identity is resolved through the same pinned context, so "info" arrives prefixed with + // the `--context ` pin this very call resolved, by name, not by endpoint: pinning + // by name (rather than an inspected endpoint) is what keeps a TLS context's own + // certificate material intact for this and every later invocation. + if (command.args[0] === "--context" && command.args[2] === "info") + return Effect.succeed(handle(0, `daemon-at-${command.args[1]}`)); + return Effect.die(`Unexpected docker command: ${command.args.join(" ")}`); + }); + const target = yield* resolveEngineTarget(spawner); + // Simulates `docker context use beta` run elsewhere, after this target was resolved. + activeContext = "beta"; + expect(contextCalls).toHaveLength(1); + expect(target.argv).toEqual(["--context", "alpha"]); + expect(target.daemonId).toBe("daemon-at-alpha"); + }).pipe( + Effect.provide( + Layer.merge(NodeServices.layer, ConfigProvider.layer(ConfigProvider.fromEnvRecord({}))), + ), + ), + ); + + it.live( + "keeps helper creation, runtime launches, password reconcile and reconcile pinned to the context an owner resolved at startup, despite a later context switch", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "engine-target-pin-" }); + const commands: Array> = []; + let activeContext = "alpha"; + const handle = (exitCode: number, stdout = "") => + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(0), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(exitCode)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + stdout: stdout === "" ? Stream.empty : Stream.succeed(new TextEncoder().encode(stdout)), + stderr: Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + // A permissive fake: every engine call this test's consumers make succeeds, so the only + // thing under test is the argv each one is actually invoked with. + const spawner = ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command) || command.command !== "docker") + return Effect.die("Unexpected command"); + commands.push(command.args); + if (command.args[0] === "context" && command.args[1] === "show") + return Effect.succeed(handle(0, `${activeContext}\n`)); + // Every other call carries the `--context ` pin this test's resolution produced. + const unpinned = command.args.slice(2); + if (unpinned[0] === "info") return Effect.succeed(handle(0, "fake-daemon-id")); + if (unpinned[0] === "version") return Effect.succeed(handle(0, "27.0.0|27.0.0")); + if (unpinned[0] === "inspect") + return Effect.succeed(handle(0, '{"Ports":{},"Status":"running","ExitCode":0}')); + if (unpinned[0] === "run") return Effect.succeed(handle(0, "abcdef012345")); + return Effect.succeed(handle(0)); + }); + + // Resolved once, as an owner's own startup would. + const target = yield* resolveEngineTarget(spawner); + expect(target.argv).toEqual(["--context", "alpha"]); + // Simulates `docker context use beta` run elsewhere, after this target was resolved. + activeContext = "beta"; + // Only the resolution above may run unpinned; every consumer exercised below must carry + // the context it pinned, regardless of the later switch. + commands.length = 0; + + const container = yield* makeContainerRuntime({ + claims: noContainerClaims, + target, + root, + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)); + + // Runtime launches. + yield* container + .prepare(image) + .pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)); + yield* Effect.scoped( + container.launchCommand({ + image, + stackId: "f".repeat(64), + instanceId: "engine-target-pin-launch", + env: {}, + args: ["-e", "process.exit(0)"], + }), + ).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)); + + // Helper creation, sharing the same container runtime a real stack would. + const storage = yield* makeDockerDatabaseStorage({ + claims: noContainerClaims, + runtime: "docker", + target, + stackId: "engine-target-pin", + instanceId: "database", + instanceRoot: root, + root, + cacheRoot: path.join(root, "cache"), + fs, + path, + crypto, + container, + spawner, + }); + yield* storage.prepare("17"); + + // Password reconcile: the consumer F2 found spawning the engine unpinned. + yield* reconcileContainerPassword( + target, + "password-reconcile-container", + Redacted.make("engine-target-pin-password"), + spawner, + ); + + // Reconcile: an orphaned claim from a crashed owner, removed through the same target. + const orphanId = "orphan00000000000000000000000000001"; + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root: path.join(root, "state") })), + StackNamespace.Service, + ); + const saved = { + id: "engine-target-pin", + runtime: "docker" as const, + identity: { projectRoot: root, branchContext: "main", stackName: "engine-target-pin" }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + yield* state.claim(saved.id, { + kind: "container", + id: orphanId, + daemonId: target.daemonId, + }); + yield* Owner.sweepContainers(state, saved, root, target).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + expect(yield* state.readClaims(saved.id)).toEqual([]); + + // Every call these consumers made, after the target was resolved, carries the context + // pinned at startup: asserted of the whole list, not merely of a filtered subset, so a + // consumer that forgets the pin (as the password reconcile once did) fails this test + // instead of being silently excluded from it. + expect(commands.length).toBeGreaterThan(0); + for (const args of commands) expect(args.slice(0, 2)).toEqual(["--context", "alpha"]); + // Each consumer is actually exercised, not vacuously skipped. + expect(commands.some((args) => args.includes("pull"))).toBe(true); + expect(commands.some((args) => args.includes("version"))).toBe(true); + expect(commands.some((args) => args.includes("password-reconcile-container"))).toBe(true); + expect(commands.some((args) => args.includes("rm") && args.includes(orphanId))).toBe(true); + }).pipe( + Effect.provide( + Layer.merge(NodeServices.layer, ConfigProvider.layer(ConfigProvider.fromEnvRecord({}))), + ), + ), + ); }); const repoDigest = (spawner: ChildProcessSpawnerService["Service"], image: string) => @@ -1366,9 +1692,9 @@ const hostGatewayRejectionScript = `console.error(${JSON.stringify( 'Error response from daemon: invalid IP address in add-host: "host-gateway"', )}); process.exit(125)`; -const podmanHostsScript = `console.log("10.88.0.1\\thost.containers.internal host.docker.internal")`; +const engineHostsScript = `console.log("10.88.0.1\\thost.docker.internal")`; -/** Rejects a `docker create` that maps the host alias to `host-gateway`, as older Podman does. */ +/** Rejects a `docker create` that maps the host alias to `host-gateway`. */ const makeHostGatewayRejectingCreateSpawner = (delegate: ChildProcessSpawnerService["Service"]) => ChildProcessSpawner.make((command) => ChildProcess.isStandardCommand(command) && @@ -1499,7 +1825,9 @@ const makeRemoveFailureSpawner = ( ChildProcess.isStandardCommand(command) && command.command === "docker" && command.args[0] === "ps" && - command.args.some((arg) => arg.startsWith("name=^/?")) + // Excludes the host-gateway probe's own throwaway "-task-" container, created and awaited + // absent during `launch`, before this test's own instance container ever reaches `remove`. + command.args.some((arg) => arg.startsWith("name=^/?") && !arg.includes("-task-")) ) { return Effect.gen(function* () { if (!(yield* Ref.get(failProbe))) return yield* delegate.spawn(command); @@ -1543,7 +1871,9 @@ const makePendingRemoveSpawner = ( ChildProcess.isStandardCommand(command) && command.command === "docker" && command.args[0] === "ps" && - command.args.some((arg) => arg.startsWith("name=^/?")) + // Excludes the host-gateway probe's own throwaway "-task-" container, created and awaited + // absent during `launch`, before this test's own instance container ever reaches `remove`. + command.args.some((arg) => arg.startsWith("name=^/?") && !arg.includes("-task-")) ) { return Effect.gen(function* () { const remaining = yield* Ref.get(pendingProbes); @@ -1690,14 +2020,14 @@ const makePendingCreateSpawner = ( }); }); -const successfulHandle = () => +const successfulHandle = (stdout = "") => ChildProcessSpawner.makeHandle({ pid: ChildProcessSpawner.ProcessId(0), exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), isRunning: Effect.succeed(false), kill: () => Effect.void, stdin: Sink.drain, - stdout: Stream.empty, + stdout: stdout === "" ? Stream.empty : Stream.succeed(new TextEncoder().encode(stdout)), stderr: Stream.empty, all: Stream.empty, getInputFd: () => Sink.drain, @@ -1711,6 +2041,7 @@ const makeHangingCreateSpawner = Effect.fn("ContainerTest.hangingCreateSpawner") const spawner = ChildProcessSpawner.make((command) => { if (!ChildProcess.isStandardCommand(command)) return Effect.die("unexpected piped command"); commands.push([...command.args]); + if (command.args[0] === "info") return Effect.succeed(successfulHandle("fake-daemon-id")); if (command.args[0] !== "create") return Effect.succeed(successfulHandle()); // The engine never answers create; the real daemon may or may not have committed it. return Deferred.succeed(createStarted, undefined).pipe( diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index d9325b6e0e..3979097fb4 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -1,6 +1,7 @@ import { withAttemptCount } from "../internal/attempts.ts"; import { Cause, + Config, Crypto, Data, Deferred, @@ -21,6 +22,8 @@ import { } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { testRunLabelArgs as readTestRunLabelArgs } from "../internal/test-run-label.ts"; +import type * as Claims from "../namespace/Claims.ts"; +import { CONTAINER_ENV_DIRNAME } from "../namespace/Paths.ts"; import { identifyContainer } from "./ContainerName.ts"; export class ContainerError extends Data.TaggedError("ContainerError")<{ @@ -115,10 +118,10 @@ const transientPullFailure = (error: ContainerError) => /** * Matches an engine CLI that is missing or reports a daemon that is not listening, not one that - * rejects the caller. Podman's connection wrappers and Windows' `error during connect` also wrap - * authentication and TLS failures, so only their refused or missing-endpoint causes match. + * rejects the caller. Windows' `error during connect` also wraps authentication and TLS failures, + * so only its refused or missing-endpoint causes match. */ -const engineUnreachable = (error: ContainerError) => +export const engineUnreachable = (error: ContainerError) => (error.cause instanceof PlatformError.PlatformError && error.cause.reason._tag === "NotFound" && error.cause.reason.method === "spawn") || @@ -126,19 +129,96 @@ const engineUnreachable = (error: ContainerError) => error.message, ); +/** Runs one engine CLI invocation outside any pinned target, for resolving that target itself. */ +const runRaw = ( + spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], + args: ReadonlyArray, +): Effect.Effect => + Effect.scoped( + Effect.gen(function* () { + const child = yield* spawner.spawn( + ChildProcess.make("docker", args, { stdin: "ignore", stdout: "pipe", stderr: "pipe" }), + ); + const [stdout, stderr, code] = yield* Effect.all( + [ + child.stdout.pipe(Stream.decodeText, Stream.mkString), + child.stderr.pipe(Stream.decodeText, Stream.mkString), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + if (Number(code) !== 0) + return yield* errorFor(args[0] ?? "command", stderr.trim() || `Engine exited with ${code}`); + return stdout; + }), + ).pipe( + Effect.timeout("10 seconds"), + Effect.mapError((cause) => + cause instanceof ContainerError ? cause : errorFor(args[0] ?? "command", cause), + ), + ); + +/** + * The engine endpoint and identity an owner resolves once, at startup, and pins for its entire + * lifetime: the container runtime, the storage helpers, the host-gateway probes and the + * namespace's reconcile loop all share this one target instead of each resolving (and so + * potentially disagreeing on) their own. `argv` is the explicit prefix every invocation carries + * (`--host `), since the docker CLI gives that priority over `DOCKER_HOST` and context + * switches, and since an argument, unlike an environment variable, never leaks to subprocesses a + * launched workload spawns. + * + * `engine` keeps the seam a future Podman driver would plug into; only `"docker"` is resolved + * today. + */ +export interface EngineTarget { + readonly engine: "docker"; + readonly argv: ReadonlyArray; + readonly daemonId: string; +} + +/** + * The active context's own name, pinned by name rather than by its endpoint alone: `context show` + * honours `DOCKER_CONTEXT` and the config file, and pinning by name keeps that context's own TLS + * material (CA, client certificate and key, skip-verify) intact for every later invocation, which + * extracting just its endpoint would otherwise drop. + */ +const resolveContextName = (spawner: ChildProcessSpawner.ChildProcessSpawner["Service"]) => + Effect.gen(function* () { + const name = (yield* runRaw(spawner, ["context", "show"])).trim(); + if (name.length === 0) return yield* errorFor("context", "Engine returned no active context"); + return name; + }); + +/** + * Resolves and pins, once for an owner's whole lifetime, the single docker endpoint its commands + * target and that endpoint's own identity. + */ +export const resolveEngineTarget = Effect.fn("Container.resolveEngineTarget")(function* ( + spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], +): Effect.fn.Return { + const host = yield* Config.option(Config.string("DOCKER_HOST")).pipe( + Effect.map(Option.filter((value) => value.length > 0)), + Effect.orElseSucceed(() => Option.none()), + ); + const argv: ReadonlyArray = Option.isSome(host) + ? ["--host", host.value] + : ["--context", yield* resolveContextName(spawner)]; + const daemonId = (yield* runRaw(spawner, [...argv, "info", "--format", "{{.ID}}"])).trim(); + if (daemonId.length === 0) return yield* errorFor("identity", "Engine returned an empty id"); + return { engine: "docker", argv, daemonId }; +}); + /** A pull worth retrying: rate-limited or a dropped connection, never an unreachable engine. */ const retryablePull = (error: ContainerError) => (rateLimited(error) || transientPullFailure(error)) && !engineUnreachable(error); -const shellQuote = (value: string): string => `'${value.replaceAll("'", "'\\''")}'`; - const PULL_MAX_RETRIES = 4; /** * Host alias mapped in Docker containers' `/etc/hosts` to the engine's IPv4 host gateway, or to * `host-gateway` for runtimes that do not await the probe; Docker Desktop's `host.docker.internal` - * and `host-gateway` also resolve to an IPv6 address. Engines that reject `host-gateway` get the - * IPv4 address they map to `host.docker.internal` or `host.containers.internal` instead. + * and `host-gateway` also resolve to an IPv6 address. An engine that rejects `host-gateway` gets + * the IPv4 address it maps to `host.docker.internal` instead. */ export const DOCKER_HOST_ALIAS = "host.supabase.internal"; @@ -147,8 +227,8 @@ const HOST_GATEWAY_PROBE_TIMEOUT: Duration.Input = "15 seconds"; const IPV4_ADDRESS = /^(?:\d{1,3}\.){3}\d{1,3}$/u; -/** Names an engine may write into `/etc/hosts` for its host, such as Podman's compat socket. */ -const ENGINE_HOST_NAMES = ["host.docker.internal", "host.containers.internal"]; +/** The name the engine writes into `/etc/hosts` for its own host. */ +const ENGINE_HOST_NAMES = ["host.docker.internal"]; const firstIpv4For = (hosts: string, names: ReadonlyArray) => hosts @@ -159,7 +239,7 @@ const firstIpv4For = (hosts: string, names: ReadonlyArray) => IPV4_ADDRESS.test(address ?? "") && mapped.some((name) => names.includes(name)), )?.[0]; -/** Matches an engine that rejects the `host-gateway` keyword in `--add-host`, such as older Podman. */ +/** Matches an engine that rejects the `host-gateway` keyword in `--add-host`. */ const rejectsHostGateway = (error: ContainerError) => /(?:invalid|unknown|unsupported|bad)[^\n]*add-host[^\n]*host-gateway/iu.test(error.message); @@ -268,7 +348,7 @@ const mountField = (key: string, value: string) => { * with backoff. */ export const makeContainerRuntime = (options: { - readonly engine: "docker" | "podman"; + readonly target: EngineTarget; readonly root: string; readonly imageMirrors?: (image: string) => ReadonlyArray; /** Omitted gives this runtime its own host-gateway probe. */ @@ -278,6 +358,11 @@ export const makeContainerRuntime = (options: { * for containers that do not rely on reaching the host over IPv4. */ readonly awaitHostGateway?: boolean; + /** + * Journals each container's exact identity before it is created, so a crashed owner leaves it + * recoverable; required so a container creation that forgets to wire claims does not compile. + */ + readonly claims: Claims.ContainerClaims; }): Effect.Effect< ContainerRuntime, never, @@ -293,6 +378,9 @@ export const makeContainerRuntime = (options: { const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; const stackRoot = path.resolve(options.root); + // An owned directory for per-launch environment files, never the real system temp directory. + const containerEnvRoot = path.join(stackRoot, CONTAINER_ENV_DIRNAME); + const engine = options.target.engine; const command = ( args: ReadonlyArray, @@ -300,14 +388,18 @@ export const makeContainerRuntime = (options: { readonly stdin?: "ignore" | "pipe"; readonly forceKillAfter?: Duration.Input; } = {}, - ) => ChildProcess.make(options.engine, args, { stdin: "ignore", ...commandOptions }); + ) => + ChildProcess.make(engine, [...options.target.argv, ...args], { + stdin: "ignore", + ...commandOptions, + }); const run = Effect.fn("Container.command")(function* ( args: ReadonlyArray, commandOptions: { readonly timeout?: Duration.Input } = { timeout: "30 seconds" }, ) { yield* Effect.annotateCurrentSpan({ - "process.executable.name": options.engine, + "process.executable.name": engine, "process.arg_count": args.length, "container.command": args[0] ?? "", }); @@ -410,34 +502,69 @@ export const makeContainerRuntime = (options: { ); const hostGateway = options.hostGateway ?? (yield* makeHostGateway); - /** Reads `/etc/hosts` from a throwaway container of an already present image. */ - const readProbeHosts = (image: string, spec: ContainerSpec, addHost: ReadonlyArray) => - testRunLabelArgs.pipe( - Effect.flatMap((testRunLabel) => - run( - [ - "run", - "--rm", - "--pull", - "never", - ...addHost, - // No instance label: `--rm` removal is asynchronous and must not count as an - // instance container; the stack labels keep it sweepable. - "--label", - `com.supabase.stack=${spec.stackId}`, - "--label", - `com.supabase.stack-root=${stackRoot}`, - ...testRunLabel, - "--entrypoint", - "cat", - image, - "/etc/hosts", - ], - { timeout: undefined }, - ), - ), - Effect.timeout(HOST_GATEWAY_PROBE_TIMEOUT), + /** Polls until the engine confirms a `--rm` container's own asynchronous removal finished. */ + const awaitRemoved = (name: string) => + run( + [ + "ps", + "--all", + "--no-trunc", + "--filter", + // Docker matches this as a regex; `.` is the only metacharacter a name can hold. + `name=^/?${name.replaceAll(".", "\\.")}$`, + "--format", + "{{.State}}", + ], + { timeout: "5 seconds" }, + ).pipe( + Effect.repeat({ + schedule: Schedule.spaced("250 millis"), + while: (output) => output !== "", + }), + Effect.timeout("10 seconds"), + Effect.asVoid, ); + /** Reads `/etc/hosts` from a throwaway, claimed container of an already present image. */ + const readProbeHosts = (image: string, spec: ContainerSpec, addHost: ReadonlyArray) => + Effect.gen(function* () { + const testRunLabel = yield* testRunLabelArgs; + const token = yield* crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => errorFor("identity", cause)), + ); + // Not an instance container: no instance label, so it is never mistaken for one. It is + // still named and claimed before the engine creates it, so a crashed owner's reconcile + // loop can still find and remove it instead of leaking it. + const { name } = identifyContainer(spec, token, true); + yield* options.claims + .claim(name, options.target.daemonId) + .pipe(Effect.mapError((cause) => errorFor("claim", cause))); + const hosts = yield* run( + [ + "run", + "--rm", + "--pull", + "never", + "--name", + name, + ...addHost, + "--label", + `com.supabase.stack=${spec.stackId}`, + "--label", + `com.supabase.stack-root=${stackRoot}`, + ...testRunLabel, + "--entrypoint", + "cat", + image, + "/etc/hosts", + ], + { timeout: undefined }, + ); + yield* awaitRemoved(name); + yield* options.claims + .unclaim(name) + .pipe(Effect.mapError((cause) => errorFor("claim", cause))); + return hosts; + }).pipe(Effect.timeout(HOST_GATEWAY_PROBE_TIMEOUT)); /** Resolves the IPv4 host address the engine writes itself, since it rejects `host-gateway`. */ const engineHostProbe = (image: string, spec: ContainerSpec, rejection: ContainerError) => readProbeHosts(image, spec, []).pipe( @@ -453,7 +580,7 @@ export const makeContainerRuntime = (options: { ? Effect.fail( new ContainerError({ operation: "host-gateway", - message: `The container engine rejects host-gateway in --add-host and maps no IPv4 address to ${ENGINE_HOST_NAMES.join(" or ")}; upgrade Podman or use --runtime podman`, + message: `The container engine rejects host-gateway in --add-host and maps no IPv4 address to ${ENGINE_HOST_NAMES.join(" or ")}`, cause: rejection, }), ) @@ -504,8 +631,11 @@ export const makeContainerRuntime = (options: { if (!Number.isInteger(port) || port < 1 || port > 65535) return yield* errorFor("ports", "Invalid container port"); } + yield* fs + .makeDirectory(containerEnvRoot, { recursive: true, mode: 0o700 }) + .pipe(Effect.mapError((cause) => errorFor("environment", cause))); const directory = yield* fs - .makeTempDirectoryScoped({ prefix: "supabase-container-" }) + .makeTempDirectoryScoped({ directory: containerEnvRoot, prefix: "container-" }) .pipe(Effect.mapError((cause) => errorFor("environment", cause))); const envPath = path.join(directory, "environment"); yield* fs @@ -521,8 +651,14 @@ export const makeContainerRuntime = (options: { Effect.mapError((cause) => errorFor("identity", cause)), ); const { name, composeProject, composeService } = identifyContainer(spec, token, oneOff); - const hostAlias = - options.engine === "docker" ? yield* hostAliasTarget(image, spec) : undefined; + // Journaled before the engine creates anything, so a crashed owner leaves an exact, + // recoverable identity instead of one the next owner must guess by label. The daemon id + // lets a later reconcile against a different daemon keep this claim rather than remove (or + // mistake) an identity that daemon has never heard of. + yield* options.claims + .claim(name, options.target.daemonId) + .pipe(Effect.mapError((cause) => errorFor("claim", cause))); + const hostAlias = yield* hostAliasTarget(image, spec); const testRunLabel = yield* testRunLabelArgs; const createArgs = (target: string | undefined) => [ "create", @@ -705,6 +841,11 @@ export const makeContainerRuntime = (options: { yield* run(["rm", name]).pipe(Effect.catchTag("ContainerError", reconcileAbsent)); yield* Ref.set(stopped, true); yield* Ref.set(removed, true); + // Cleanup authority now belongs entirely to this normal teardown; a crash before this + // point still leaves the claim for the next owner's reconcile loop. + yield* options.claims + .unclaim(name) + .pipe(Effect.mapError((cause) => errorFor("claim", cause))); }); yield* Scope.addFinalizer( owner, @@ -836,16 +977,10 @@ export const makeContainerRuntime = (options: { (yield* Effect.gen(function* () { // Released on exit, since a release left for service stop can hit a reused pid. const followerScope = yield* Scope.fork(owner); - const follower = yield* spawner - .spawn( - ChildProcess.make(options.engine, ["logs", "--follow", name], { - stdin: "ignore", - }), - ) - .pipe( - Scope.provide(followerScope), - Effect.mapError((cause) => errorFor("logs", cause)), - ); + const follower = yield* spawner.spawn(command(["logs", "--follow", name])).pipe( + Scope.provide(followerScope), + Effect.mapError((cause) => errorFor("logs", cause)), + ); yield* Effect.forkIn( Effect.exit(follower.exitCode).pipe( Effect.andThen(Scope.close(followerScope, Exit.void)), @@ -875,102 +1010,46 @@ export const makeContainerRuntime = (options: { }; }); -export const removeStackContainers = Effect.fn("Container.removeStackContainers")( - (options: { - readonly engine: "docker" | "podman"; - readonly stackId: string; - readonly root: string; - }) => +/** + * Removes a container by its exact recorded identity, succeeding when it is already absent. The + * namespace's reconcile loop uses this instead of discovering candidates by label, through the + * same pinned {@link EngineTarget} it resolved identity through, so a context switch in between + * can never split the two. + */ +export const removeContainerById = Effect.fn("Container.removeContainerById")( + (options: { readonly target: EngineTarget; readonly id: string }) => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const stackRoot = options.root; - const run = Effect.fn("Container.runCleanupCommand")(function* (args: ReadonlyArray) { - return yield* Effect.scoped( - Effect.gen(function* () { - const child = yield* spawner.spawn( - ChildProcess.make(options.engine, args, { - stdin: "ignore", - stdout: "pipe", - stderr: "pipe", - }), - ); - const [stdout, stderr, code] = yield* Effect.all( - [ - child.stdout.pipe(Stream.decodeText, Stream.mkString), - child.stderr.pipe(Stream.decodeText, Stream.mkString), - child.exitCode, - ], - { concurrency: "unbounded" }, - ); - if (Number(code) !== 0) - return yield* errorFor( - args[0] ?? "cleanup", - stderr.trim() || `Engine exited with ${code}`, - ); - return stdout.trim(); - }), - ).pipe( - Effect.timeout("30 seconds"), - Effect.mapError((cause) => - cause instanceof ContainerError ? cause : errorFor(args[0] ?? "cleanup", cause), - ), - ); - }); - const filters = [ - "--filter", - `label=com.supabase.stack=${options.stackId}`, - "--filter", - `label=com.supabase.stack-root=${stackRoot}`, - ]; - const list = () => run(["ps", "--all", "--quiet", "--no-trunc", ...filters]); - // Only the initial listing can show the engine itself is unreachable; a later `rm` or - // leftover check failing is a per-container cleanup problem instead. - const ids = (yield* list().pipe( - Effect.mapError((cause) => - engineUnreachable(cause) + return yield* Effect.scoped( + Effect.gen(function* () { + const child = yield* spawner.spawn( + ChildProcess.make( + options.target.engine, + [...options.target.argv, "rm", "--force", options.id], + { stdin: "ignore", stdout: "pipe", stderr: "pipe" }, + ), + ); + const [stderr, code] = yield* Effect.all( + [child.stderr.pipe(Stream.decodeText, Stream.mkString), child.exitCode], + { concurrency: "unbounded" }, + ); + // Docker reports this exact phrasing for an id that no longer exists. + if (Number(code) !== 0 && !/no such container/iu.test(stderr)) + return yield* errorFor("cleanup", stderr.trim() || `Engine exited with ${code}`); + }), + ).pipe( + Effect.timeout("30 seconds"), + Effect.mapError((cause) => { + const error = cause instanceof ContainerError ? cause : errorFor("cleanup", cause); + return engineUnreachable(error) ? new ContainerError({ - operation: cause.operation, - message: cause.message, - cause: cause.cause, + operation: error.operation, + message: error.message, + cause: error.cause, reason: "engine-unavailable", }) - : cause, - ), - )) - .split("\n") - .filter((id) => id.length > 0); - yield* Effect.forEach( - ids, - (id) => - run(["rm", "--force", id]).pipe( - Effect.catchTag("ContainerError", (failure) => - run(["ps", "--all", "--quiet", "--no-trunc", "--filter", `id=${id}`]).pipe( - Effect.flatMap((present) => - present.length === 0 ? Effect.void : Effect.fail(failure), - ), - ), - ), - ), - { concurrency: 1, discard: true }, + : error; + }), ); - const remaining = yield* list(); - if (remaining.length > 0) - return yield* errorFor("cleanup", `Stack containers remain: ${remaining}`); }), ); - -/** Shell command that removes the same containers as `removeStackContainers`, succeeding when none remain. */ -export const removeStackContainersCommand = (options: { - readonly engine: "docker" | "podman"; - readonly stackId: string; - readonly root: string; -}): string => { - const filters = [ - `label=com.supabase.stack=${options.stackId}`, - `label=com.supabase.stack-root=${options.root}`, - ] - .map((filter) => `--filter ${shellQuote(filter)}`) - .join(" "); - // `sh -c` keeps POSIX word splitting of `$ids` when pasted into shells like zsh that skip it. - return `sh -c ${shellQuote(`ids=$(${options.engine} ps --all --quiet --no-trunc ${filters}) && { [ -z "$ids" ] || ${options.engine} rm --force $ids; }`)}`; -}; diff --git a/packages/stack/src/runtime/NativeProcess.ts b/packages/stack/src/runtime/NativeProcess.ts index 1826d5ca67..bd078f7001 100644 --- a/packages/stack/src/runtime/NativeProcess.ts +++ b/packages/stack/src/runtime/NativeProcess.ts @@ -3,6 +3,7 @@ import { fileURLToPath } from "node:url"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import type { PlatformError } from "effect/PlatformError"; import { isBunVirtualPath } from "../internal/dispatch-markers.ts"; +import * as Environment from "../namespace/Environment.ts"; import type { ChildProcessHandle, ExitCode, @@ -12,7 +13,10 @@ import type { export interface NativeProcessSpec { readonly executable: string; readonly args?: ReadonlyArray; + /** Merged over {@link environment}'s confined values; it is rejected if it sets any of them. */ readonly env?: Readonly>; + /** The owned HOME/TMPDIR/XDG/Deno directories this workload is confined to. */ + readonly environment: Environment.NativeEnvironment; readonly cwd?: string; readonly stdin?: "ignore" | "pipe"; /** Numeric identity the workload runs as; omitted keeps the launcher's identity. */ @@ -71,7 +75,7 @@ export const defaultNativeProcessLauncher = (): NativeProcessLauncher => ({ args: [nativeLauncherEntrypointFor(import.meta.url)], }); -const encodeSpec = (spec: NativeProcessSpec): Uint8Array => { +const encodeSpec = (spec: NativeProcessSpec, env: Readonly>): Uint8Array => { const timeout = spec.gracefulStopSignal === undefined ? Option.none() @@ -84,7 +88,7 @@ const encodeSpec = (spec: NativeProcessSpec): Uint8Array => { JSON.stringify({ executable: spec.executable, args: spec.args ?? [], - env: spec.env, + env, cwd: spec.cwd, uid: spec.uid, gid: spec.gid, @@ -123,6 +127,7 @@ export const spawnNativeProcess = Effect.fn("NativeProcess.spawn")(function* ( "process.executable.name": spec.executable.split(/[\\/]/u).pop() ?? spec.executable, "process.arg_count": spec.args?.length ?? 0, }); + const env = yield* Environment.apply(spec.environment, spec.env); return yield* Effect.gen(function* () { // Shutdown must precede the spawner's finalizer even when the caller closes in parallel. const processScope = yield* Scope.fork(yield* Scope.Scope, "sequential"); @@ -344,7 +349,7 @@ export const spawnNativeProcess = Effect.fn("NativeProcess.spawn")(function* ( Effect.orDie, ), ); - yield* Stream.run(Stream.succeed(encodeSpec(spec)), handle.getInputFd(4)); + yield* Stream.run(Stream.succeed(encodeSpec(spec, env)), handle.getInputFd(4)); const groupIdLine = yield* handle .getOutputFd(5) .pipe(Stream.decodeText, Stream.splitLines, Stream.runHead); diff --git a/packages/stack/src/runtime/container-image-mirror.integration.test.ts b/packages/stack/src/runtime/container-image-mirror.integration.test.ts index e528d9d33e..a739c40662 100644 --- a/packages/stack/src/runtime/container-image-mirror.integration.test.ts +++ b/packages/stack/src/runtime/container-image-mirror.integration.test.ts @@ -3,8 +3,11 @@ import { describe, expect, it } from "@effect/vitest"; import { Cause, Effect, Exit, Fiber, Layer, Option, Sink, Stream } from "effect"; import { TestClock } from "effect/testing"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { makeContainerRuntime } from "./Container.ts"; +import { makeContainerRuntime, type EngineTarget } from "./Container.ts"; +import { noContainerClaims } from "../../tests/claims.ts"; +// An unpinned target: these tests exercise image mirroring, not endpoint pinning. +const dockerTarget: EngineTarget = { engine: "docker", argv: [], daemonId: "test-daemon-id" }; const primary = "ghcr.io/supabase/cli/postgrest:v16.2"; const mirror = "public.ecr.aws/supabase/cli/postgrest:v16.2"; const secondMirror = "registry.test/supabase/cli/postgrest:v16.2"; @@ -45,6 +48,9 @@ const fakeEngine = (options: { const args = [...command.args]; commands.push(args); const ref = args.at(-1) ?? ""; + // Resolved once before any container is created; this stand-in always reports one so launch + // never fails at that earlier step for reasons unrelated to what each test exercises. + if (args[0] === "info") return Effect.succeed(handle(0, "fake-daemon-id")); if (args[0] === "image") return Effect.succeed(handle(0, local.has(ref) ? "sha256:1" : "")); if (args[0] === "pull") { const queued = failures.get(ref); @@ -79,7 +85,8 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [mirror] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - engine: "docker", + claims: noContainerClaims, + target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror] : []), }); @@ -97,7 +104,8 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [secondMirror] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - engine: "docker", + claims: noContainerClaims, + target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror, secondMirror] : []), }); @@ -116,7 +124,8 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [mirror] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - engine: "docker", + claims: noContainerClaims, + target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror] : []), }); @@ -133,7 +142,8 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [mirror] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - engine: "docker", + claims: noContainerClaims, + target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror] : []), }); @@ -150,7 +160,8 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - engine: "docker", + claims: noContainerClaims, + target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror] : []), }); @@ -168,7 +179,8 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [primary, mirror] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - engine: "docker", + claims: noContainerClaims, + target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror] : []), }); @@ -181,7 +193,11 @@ describe("container image mirror", () => { it.effect("retries a rate-limited pull with backoff until the registry accepts it", () => { const engine = fakeEngine({ pullable: [primary], throttled: { [primary]: 2 } }); return Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); const prepared = yield* runtime.prepare(primary).pipe(Effect.forkChild); yield* TestClock.adjust("1 minute"); yield* Fiber.join(prepared); @@ -198,7 +214,11 @@ describe("container image mirror", () => { }, }); return Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); const prepared = yield* runtime.prepare(primary).pipe(Effect.forkChild); yield* TestClock.adjust("1 minute"); yield* Fiber.join(prepared); @@ -220,7 +240,11 @@ describe("container image mirror", () => { failures: { [primary]: [message] }, }); return Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); const failed = yield* runtime.prepare(primary).pipe(Effect.exit); const error = Exit.isFailure(failed) ? Option.getOrUndefined(Cause.findErrorOption(failed.cause)) @@ -234,7 +258,11 @@ describe("container image mirror", () => { it.effect("reports the rate limit after five throttled attempts", () => { const engine = fakeEngine({ pullable: [primary], throttled: { [primary]: 10 } }); return Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); const prepared = yield* runtime.prepare(primary).pipe(Effect.exit, Effect.forkChild); yield* TestClock.adjust("5 minutes"); const failed = yield* Fiber.join(prepared); @@ -249,7 +277,11 @@ describe("container image mirror", () => { it.effect("stops pulling once a concurrent prepare lands the image during backoff", () => { const engine = fakeEngine({ pullable: [primary], throttled: { [primary]: 1 } }); return Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: ".", + }); const throttled = yield* runtime.prepare(primary).pipe(Effect.forkChild); yield* TestClock.adjust("1 millis"); yield* runtime.prepare(primary); @@ -263,7 +295,8 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [mirror], throttled: { [primary]: 10 } }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - engine: "docker", + claims: noContainerClaims, + target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror] : []), }); diff --git a/packages/stack/src/runtime/native-process.integration.test.ts b/packages/stack/src/runtime/native-process.integration.test.ts index 0fef4834c0..4f86409acb 100644 --- a/packages/stack/src/runtime/native-process.integration.test.ts +++ b/packages/stack/src/runtime/native-process.integration.test.ts @@ -10,6 +10,7 @@ import type { ExitCode } from "effect/unstable/process/ChildProcessSpawner"; import { execFileSync, spawn as spawnProcess } from "node:child_process"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- The test waits for actual inherited-fd and process events. import { once } from "node:events"; +import type { NativeEnvironment } from "../namespace/Environment.ts"; import { defaultNativeProcessLauncher, spawnNativeProcess, @@ -19,6 +20,21 @@ import { const targetPid = 87_035; +/** These tests exercise process lifecycle, not confinement; the paths need not exist. */ +const testEnvironment: NativeEnvironment = { + values: { + HOME: "/tmp/native-process-test-home", + TMPDIR: "/tmp/native-process-test-home/tmp", + TMP: "/tmp/native-process-test-home/tmp", + TEMP: "/tmp/native-process-test-home/tmp", + XDG_CACHE_HOME: "/tmp/native-process-test-home/.cache", + XDG_CONFIG_HOME: "/tmp/native-process-test-home/.config", + XDG_DATA_HOME: "/tmp/native-process-test-home/.local/share", + XDG_STATE_HOME: "/tmp/native-process-test-home/.local/state", + DENO_DIR: "/tmp/native-process-test-home/.cache/deno", + }, +}; + interface FakeProcessOptions { readonly groupOutput?: string; readonly groupExitCode?: number; @@ -133,7 +149,7 @@ const makeSpawner = (options: FakeProcessOptions) => { return spawner; }; -const spec: NativeProcessSpec = { executable: "test-native-process" }; +const spec: NativeProcessSpec = { executable: "test-native-process", environment: testEnvironment }; // Intercepts every signal sent to the positive launcher pid and its negative // group so tests using a placeholder pid never reach a real OS process, for @@ -246,6 +262,7 @@ const descendantSpec = (): NativeProcessSpec => { return { executable: process.execPath, args: ["--input-type=module", "-e", workloadCode], + environment: testEnvironment, }; }; @@ -468,6 +485,7 @@ describe("native process group cleanup", () => { Effect.gen(function* () { const native = yield* spawnNativeProcess({ executable: process.execPath, + environment: testEnvironment, stdin: "pipe", args: [ "--input-type=module", @@ -500,6 +518,7 @@ describe("native process group cleanup", () => { Effect.gen(function* () { const native = yield* spawnNativeProcess({ executable: process.execPath, + environment: testEnvironment, args: [ "--input-type=module", "-e", @@ -717,6 +736,7 @@ describe("native process group cleanup", () => { const realSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const native = yield* spawnNativeProcess({ executable: process.execPath, + environment: testEnvironment, args: [ "--input-type=module", "-e", @@ -793,7 +813,10 @@ describe("native process group cleanup", () => { it.live("reports a native launcher startup failure when workload spawn fails", () => Effect.scoped( - spawnNativeProcess({ executable: "/definitely/missing/native-workload" }).pipe(Effect.exit), + spawnNativeProcess({ + executable: "/definitely/missing/native-workload", + environment: testEnvironment, + }).pipe(Effect.exit), ).pipe( Effect.provide(NodeServices.layer), Effect.tap((result) => @@ -936,6 +959,7 @@ describe("native process group cleanup", () => { spawnNativeProcess( { executable: process.execPath, + environment: testEnvironment, args: [ "-e", "process.on('SIGTERM', () => {}); process.stdout.write('native-workload-ready\\n'); setInterval(() => {}, 1000)", diff --git a/packages/stack/src/runtime/postgres-user.integration.test.ts b/packages/stack/src/runtime/postgres-user.integration.test.ts new file mode 100644 index 0000000000..a6f867c3d2 --- /dev/null +++ b/packages/stack/src/runtime/postgres-user.integration.test.ts @@ -0,0 +1,147 @@ +import { NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Layer, Path } from "effect"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- the current process's own uid/gid make the chown calls below harmless no-ops. +import { userInfo } from "node:os"; +import { + acquireNativeRuntimeRoot, + handOverNativePostgresFiles, + NativeRuntimeRootBase, + nativeRuntimeRootPath, + type PasswdEntry, +} from "./postgres-user.ts"; + +/** + * Exercises only the symlink-rejection step of the step-down chown, which needs no privilege: the + * handed-off targets are chowned to the current process's own uid/gid, a harmless no-op, so the + * test runs identically whether or not it has root. The cross-uid chown itself (the part that + * actually needs root) is verified by trace, not by this test: `handOverNativePostgresFiles` + * chowns the confined HOME once, with `-R -P` and no nested paths as separate arguments (see + * `chownRecursive` and its single call per target in `runtime/postgres-user.ts`). + */ +describe("handOverNativePostgresFiles", () => { + it.live("refuses a confined HOME replaced by a symlink, without chowning through it", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "postgres-user-symlink-" }); + const self = userInfo(); + const user: PasswdEntry = { + name: self.username, + uid: self.uid, + gid: self.gid, + home: self.homedir, + }; + const dataPath = path.join(root, "data"); + const rootKeyPath = path.join(root, "pgsodium_root.key"); + const socketPath = path.join(root, "socket"); + const hbaPath = path.join(socketPath, "pg_hba.conf"); + const bundleRoot = path.join(root, "bundle"); + const executable = path.join(bundleRoot, "bin", "postgres"); + yield* fs.makeDirectory(dataPath, { recursive: true }); + yield* fs.makeDirectory(socketPath, { recursive: true }); + yield* fs.makeDirectory(path.dirname(executable), { recursive: true }); + yield* fs.writeFileString(rootKeyPath, "root-key"); + yield* fs.writeFileString(hbaPath, "local all all trust\n"); + yield* fs.writeFileString(executable, "#!/bin/sh\n"); + + const outside = yield* fs.makeTempDirectoryScoped({ + prefix: "postgres-user-symlink-outside-", + }); + const environmentHome = path.join(root, "home"); + yield* fs.symlink(outside, environmentHome); + + const failure = yield* handOverNativePostgresFiles(user, { + dataPath, + rootKeyPath, + socketPath, + hbaPath, + runtimeRoot: root, + bundleRoot, + executable, + environmentHome, + }).pipe(Effect.flip); + expect(failure.message).toContain("is a symlink"); + expect(yield* fs.readLink(environmentHome).pipe(Effect.isSuccess)).toBe(true); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); +}); + +describe("acquireNativeRuntimeRoot", () => { + it.live("refuses a pre-existing runtime root that is a symlink, leaving it untouched", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const base = yield* fs.makeTempDirectoryScoped({ + prefix: "native-runtime-root-symlink-", + }); + const outside = yield* fs.makeTempDirectoryScoped({ + prefix: "native-runtime-root-outside-", + }); + const root = nativeRuntimeRootPath(path, base, process.getuid?.() ?? 0); + yield* fs.symlink(outside, root); + + const failure = yield* acquireNativeRuntimeRoot().pipe( + Effect.flip, + Effect.provide(Layer.succeed(NativeRuntimeRootBase, base)), + ); + expect(failure.message).toContain("is a symlink"); + expect(yield* fs.readLink(root)).toBe(outside); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live( + "refuses a pre-existing runtime root that is group- or world-writable, leaving it untouched", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const base = yield* fs.makeTempDirectoryScoped({ + prefix: "native-runtime-root-writable-", + }); + const root = nativeRuntimeRootPath(path, base, process.getuid?.() ?? 0); + yield* fs.makeDirectory(root); + // chmod, unlike the mkdir mode above, is never narrowed by the process umask. + yield* fs.chmod(root, 0o707); + + const failure = yield* acquireNativeRuntimeRoot().pipe( + Effect.flip, + Effect.provide(Layer.succeed(NativeRuntimeRootBase, base)), + ); + expect(failure.message).toContain("writable by group or others"); + expect((yield* fs.stat(root)).mode & 0o777).toBe(0o707); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live( + "refuses a base beneath a group- or world-writable directory without the sticky bit", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const shared = yield* fs.makeTempDirectoryScoped({ + prefix: "native-runtime-root-shared-", + }); + yield* fs.chmod(shared, 0o777); + const base = path.join(shared, "runtime"); + yield* fs.makeDirectory(base, { mode: 0o700 }); + + const failure = yield* acquireNativeRuntimeRoot().pipe( + Effect.flip, + Effect.provide(Layer.succeed(NativeRuntimeRootBase, base)), + ); + expect(failure.message).toContain("writable by group or others without the sticky bit"); + expect(yield* fs.exists(nativeRuntimeRootPath(path, base, process.getuid?.() ?? 0))).toBe( + false, + ); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); +}); diff --git a/packages/stack/src/runtime/postgres-user.ts b/packages/stack/src/runtime/postgres-user.ts index b53302bee1..9d0c3f5c00 100644 --- a/packages/stack/src/runtime/postgres-user.ts +++ b/packages/stack/src/runtime/postgres-user.ts @@ -1,10 +1,25 @@ -import { Config, Effect, FileSystem, Option, Path } from "effect"; +import { Config, Context, Effect, FileSystem, Option, Path } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import type { PlatformError } from "effect/PlatformError"; +import { lstatPath } from "../namespace/drivers/FileSystem.ts"; import { ServiceError } from "../Service.ts"; /** Names the non-root system user that runs native PostgreSQL when the stack itself runs as root. */ export const NATIVE_POSTGRES_USER_ENV = "SUPABASE_NATIVE_POSTGRES_USER"; +/** + * Base directory of the per-uid native runtime root. Always the system `/tmp` outside tests: its + * protection is the administrator's, so no user-chosen base widens what recovery may delete. + */ +export const NativeRuntimeRootBase = Context.Reference( + "@supabase/stack/NativeRuntimeRootBase", + { defaultValue: () => "/tmp" }, +); + +/** The per-uid private root every native socket directory nests under. */ +export const nativeRuntimeRootPath = (path: Path.Path, base: string, uid: number): string => + path.join(base, `supabase-${uid}`); + export interface PasswdEntry { readonly name: string; readonly uid: number; @@ -122,6 +137,102 @@ const ancestorsOf = (path: Path.Path, start: string): ReadonlyArray => { return parent === start ? [start] : [start, ...ancestorsOf(path, parent)]; }; +const isAlreadyExists = (error: PlatformError): boolean => error.reason._tag === "AlreadyExists"; + +/** + * Confirms `target`'s unresolved state (one `lstat`, so a symlink is never followed) is a + * directory worth trusting: owned by `uid`, or by root when `allowRootOwner`, and never writable by + * group or others unless `allowStickyWritable` and the sticky bit guard it, as `/tmp` itself is. + */ +const confirmPrivateDirectory = Effect.fn("NativePostgresUser.confirmPrivateDirectory")(function* ( + target: string, + uid: number, + policy: { readonly allowRootOwner: boolean; readonly allowStickyWritable: boolean }, +) { + const info = yield* lstatPath(target).pipe( + Effect.mapError( + (cause) => + new ServiceError({ operation: "launch", message: `Unable to inspect ${target}`, cause }), + ), + ); + if (info?.type === "SymbolicLink") + return yield* new ServiceError({ + operation: "launch", + message: `${target} is a symlink; refusing to use it for the native runtime root`, + }); + if (info === undefined || info.type !== "Directory") + return yield* new ServiceError({ + operation: "launch", + message: `${target} is not a directory; refusing to use it for the native runtime root`, + }); + if (info.uid !== uid && !(policy.allowRootOwner && info.uid === 0)) + return yield* new ServiceError({ + operation: "launch", + message: `${target} belongs to uid ${String(info.uid)}, not the current user; refusing to use it for the native runtime root`, + }); + const writable = (info.mode & 0o022) !== 0; + const sticky = (info.mode & 0o1000) !== 0; + if (writable && !(policy.allowStickyWritable && sticky)) + return yield* new ServiceError({ + operation: "launch", + message: `${target} is writable by group or others${policy.allowStickyWritable ? " without the sticky bit" : ""}; refusing to use it for the native runtime root`, + }); + return info; +}); + +/** + * Resolves and confirms the per-uid root native socket directories nest under: the configured base, + * canonicalized with `realPath` so a symlink or replacement planted after this check can't retarget + * it, then its `supabase-` leaf, confirmed owned by this uid alone with no write bit at all. + * `create` additionally makes the leaf (mode 0700) first, tolerating a race with another of our own + * processes. + */ +const resolveNativeRuntimeRoot = (create: boolean) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const base = yield* NativeRuntimeRootBase; + const uid = process.getuid?.() ?? 0; + const canonicalBase = yield* fs + .realPath(base) + .pipe( + Effect.mapError( + (cause) => + new ServiceError({ operation: "launch", message: `Unable to resolve ${base}`, cause }), + ), + ); + // Every ancestor is checked too: a writable non-sticky one would let another uid rename the tree. + for (const directory of ancestorsOf(path, canonicalBase)) + yield* confirmPrivateDirectory(directory, uid, { + allowRootOwner: true, + allowStickyWritable: true, + }); + const root = nativeRuntimeRootPath(path, canonicalBase, uid); + if (create) + yield* fs.makeDirectory(root, { mode: 0o700 }).pipe( + Effect.catchIf(isAlreadyExists, () => Effect.void), + Effect.mapError( + (cause) => + new ServiceError({ operation: "launch", message: `Unable to create ${root}`, cause }), + ), + ); + yield* confirmPrivateDirectory(root, uid, { + allowRootOwner: false, + allowStickyWritable: false, + }); + return root; + }); + +/** Acquires this process's native runtime root for a launch, creating the leaf if missing. */ +export const acquireNativeRuntimeRoot = Effect.fn("NativePostgresUser.acquireRuntimeRoot")(() => + resolveNativeRuntimeRoot(true), +); + +/** Resolves the native runtime root for recovery's containment check, without creating it. */ +export const resolveNativeRuntimeRootForRecovery = Effect.fn( + "NativePostgresUser.resolveRuntimeRootForRecovery", +)(() => resolveNativeRuntimeRoot(false)); + /** * Restricts a directory to its owner but keeps an existing traverse-only grant, because a * stepped-down PostgreSQL resolves bundle and data paths through the cache and state roots while it runs. @@ -191,7 +302,36 @@ export const openNativePostgresInstance = Effect.fn("NativePostgresUser.openInst /** The bundle's first-boot init runs `chmod +x` on this script, which only its owner may do. */ const GETKEY_SCRIPT = "share/supabase-cli/config/pgsodium_getkey.sh"; -/** Hands the instance data, key, socket, HBA file, and the bundle's getkey script to the PostgreSQL user. */ +/** `chown -R -P uid:gid target`, failing with a message naming `target` and the recipient user. */ +const chownRecursive = Effect.fn("NativePostgresUser.chownRecursive")(function* ( + user: PasswdEntry, + target: string, +) { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + // -P never follows symlinks inside the tree, and directories change owner after their contents. + const chownArgs = ["-R", "-P", `${user.uid}:${user.gid}`, target]; + yield* Effect.annotateCurrentSpan({ + "process.executable.name": "chown", + "process.arg_count": chownArgs.length, + }); + const status = yield* spawner.exitCode( + ChildProcess.make("chown", chownArgs, { stdin: "ignore", stdout: "ignore", stderr: "ignore" }), + ); + yield* Effect.annotateCurrentSpan("process.exit_code", Number(status)); + if (Number(status) !== 0) + return yield* new ServiceError({ + operation: "launch", + message: `chown exited with ${String(status)} while handing ${target} to '${user.name}'`, + }); +}); + +/** + * Hands the instance data, key, socket, HBA file, the bundle's getkey script, and a confined + * native `environmentHome` (if any) to the PostgreSQL user, each with its own `chown -R -P` call + * so a nested directory is never also listed as a separate, redundant target. `runtimeRoot` and its + * ancestors only gain traverse, so the step-down user reaches its socket directory under a + * restrictive configured base without write access to any of them. + */ export const handOverNativePostgresFiles = Effect.fn("NativePostgresUser.handOverFiles")(function* ( user: PasswdEntry, paths: { @@ -199,13 +339,15 @@ export const handOverNativePostgresFiles = Effect.fn("NativePostgresUser.handOve readonly rootKeyPath: string; readonly socketPath: string; readonly hbaPath: string; + readonly runtimeRoot: string; readonly bundleRoot: string; readonly executable: string; + /** A confined native environment root (HOME); chowned on its own, never as a nested target. */ + readonly environmentHome?: string; }, ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const getkey = path.join(paths.bundleRoot, GETKEY_SCRIPT); const hasGetkey = yield* fs.exists(getkey); const targets = [paths.dataPath, paths.rootKeyPath, paths.socketPath, paths.hbaPath]; @@ -213,26 +355,19 @@ export const handOverNativePostgresFiles = Effect.fn("NativePostgresUser.handOve yield* requireOwnedByRootOr(user, getkey); targets.push(getkey); } - // -P never follows symlinks inside the tree, and directories change owner after their contents. - const chownArgs = ["-R", "-P", `${user.uid}:${user.gid}`, ...targets]; - yield* Effect.annotateCurrentSpan({ - "process.executable.name": "chown", - "process.arg_count": chownArgs.length, - }); - const status = yield* spawner.exitCode( - ChildProcess.make("chown", chownArgs, { - stdin: "ignore", - stdout: "ignore", - stderr: "ignore", - }), - ); - yield* Effect.annotateCurrentSpan("process.exit_code", Number(status)); - if (Number(status) !== 0) - return yield* new ServiceError({ - operation: "launch", - message: `chown exited with ${String(status)} while handing files to '${user.name}'`, - }); + for (const target of targets) yield* chownRecursive(user, target); + if (paths.environmentHome !== undefined) { + // A prior launch's confined HOME could have been replaced by a symlink or junction between + // runs; refuse to chown through it rather than trust reuse. + if ((yield* lstatPath(paths.environmentHome))?.type === "SymbolicLink") + return yield* new ServiceError({ + operation: "launch", + message: `${paths.environmentHome} is a symlink; refusing to hand it to '${user.name}'`, + }); + yield* chownRecursive(user, paths.environmentHome); + } yield* allowTraverse(user, [ + ...ancestorsOf(path, paths.runtimeRoot), ...ancestorsOf(path, path.dirname(paths.executable)), ...(hasGetkey ? ancestorsOf(path, path.dirname(getkey)) : []), ]); diff --git a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts index de79a2f854..c734cd2826 100644 --- a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts +++ b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts @@ -6,6 +6,8 @@ import { makeService } from "../Service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; +import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ stackId: "catalog-analytics", @@ -13,11 +15,14 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, }); const dockerOptions = (root: string) => ({ ...options(root), runtime: "docker" as const, + engineTarget: dockerEngineTarget, }); describe("service catalog", () => { @@ -29,6 +34,12 @@ describe("service catalog", () => { const fs = yield* FileSystem.FileSystem; const client = yield* HttpClient.HttpClient; const root = yield* makeDockerDatabaseRoot("catalog-optional-data-"); + // Ownership is by location: Vector's config file and Imgproxy's served directory are + // caller paths and must live outside the stack's data root, not merely outside each + // service's own instance root. + const callerRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "catalog-optional-data-caller-", + }); const secret = "catalog-optional-data-secret-with-at-least-32-chars"; const databaseRecipe = yield* makeServiceRecipe( { @@ -79,13 +90,13 @@ describe("service catalog", () => { config: { analyticsUrl: `http://${analyticsRelay.host}:${analyticsRelay.port}`, apiKey: "catalog-analytics", - configPath: `${root}/vector.yaml`, + configPath: `${callerRoot}/vector.yaml`, }, }, dockerOptions(root), ); yield* fs.writeFileString( - `${root}/vector.yaml`, + `${callerRoot}/vector.yaml`, "sources:\n dummy:\n type: demo_logs\n format: syslog\n interval: 60\n" + "sinks:\n print:\n type: console\n inputs: [dummy]\n encoding:\n codec: json\n", ); @@ -101,7 +112,7 @@ describe("service catalog", () => { ); expect(vectorResponse.status).toBe(200); - const imageRoot = `${root}/images`; + const imageRoot = `${callerRoot}/images`; yield* fs.makeDirectory(imageRoot, { recursive: true }); const imgproxyRecipe = yield* makeServiceRecipe( { service: "imgproxy", config: { filePath: imageRoot } }, diff --git a/packages/stack/src/services/AuthStorage.integration.test.ts b/packages/stack/src/services/AuthStorage.integration.test.ts index 437e7975e8..453562fb83 100644 --- a/packages/stack/src/services/AuthStorage.integration.test.ts +++ b/packages/stack/src/services/AuthStorage.integration.test.ts @@ -7,6 +7,8 @@ import { makeService } from "../Service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; +import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ stackId: "catalog-auth-storage", @@ -14,11 +16,14 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, }); const dockerOptions = (root: string) => ({ ...options(root), runtime: "docker" as const, + engineTarget: dockerEngineTarget, }); describe("service catalog", () => { @@ -30,6 +35,11 @@ describe("service catalog", () => { const fs = yield* FileSystem.FileSystem; const client = yield* HttpClient.HttpClient; const root = yield* makeDockerDatabaseRoot("catalog-auth-storage-"); + // Ownership is by location: Storage's filePath is a caller path and must live outside + // the stack's data root, not merely outside the service's own instance root. + const callerRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "catalog-auth-storage-caller-", + }); const secret = "catalog-auth-storage-secret-with-at-least-32-chars"; const databaseRecipe = yield* makeServiceRecipe( { @@ -93,7 +103,7 @@ describe("service catalog", () => { )(loginBody); expect(loginToken.access_token.length).toBeGreaterThan(0); - const storageRoot = `${root}/storage`; + const storageRoot = `${callerRoot}/storage`; yield* fs.makeDirectory(storageRoot, { recursive: true }); const imgproxyRecipe = yield* makeServiceRecipe( { service: "imgproxy", config: { filePath: storageRoot } }, diff --git a/packages/stack/src/services/Catalog.integration.test.ts b/packages/stack/src/services/Catalog.integration.test.ts index be06f948a4..016b8e7909 100644 --- a/packages/stack/src/services/Catalog.integration.test.ts +++ b/packages/stack/src/services/Catalog.integration.test.ts @@ -2,6 +2,7 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Effect, Exit, FileSystem, Layer, Redacted } from "effect"; import { makeServiceRecipe } from "./Catalog.ts"; +import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; const options = (root: string) => ({ stackId: "catalog-recipe", @@ -9,6 +10,8 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, }); describe("service catalog", () => { diff --git a/packages/stack/src/services/Catalog.ts b/packages/stack/src/services/Catalog.ts index 64e7828df5..7957a293ef 100644 --- a/packages/stack/src/services/Catalog.ts +++ b/packages/stack/src/services/Catalog.ts @@ -30,6 +30,7 @@ import { type ProcessRecipeResult, } from "./Recipe.ts"; import { makeProcessRecipe, type ProcessDependencies } from "./ProcessRecipe.ts"; +import { borrow } from "../namespace/Paths.ts"; import { missingInput } from "./ServiceConfig.ts"; import type { SnapshotScope } from "./DatabaseSnapshot.ts"; import { slimImageMirrors, type ServiceKind } from "../Artifacts.ts"; @@ -329,6 +330,9 @@ export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( runtime: options.runtime, ...(options.helpers === undefined ? {} : { helpers: options.helpers }), ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), + ...(options.engineTarget === undefined ? {} : { engineTarget: options.engineTarget }), + containerClaims: options.containerClaims, + directoryClaims: options.directoryClaims, }).pipe( Effect.mapError( (cause) => @@ -343,15 +347,29 @@ export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( return databaseRecipe(creation, component); } const container = - options.runtime === "native" + options.engineTarget === undefined ? undefined : yield* makeContainerRuntime({ - engine: options.runtime, + target: options.engineTarget, root: options.root, imageMirrors: slimImageMirrors, ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), + claims: options.containerClaims, }); - const deps: ProcessDependencies = { fs, path, crypto, client, spawner, container }; + const borrowCallerPath = (candidate: string) => + borrow(fs, path, candidate, options.root, (operation, cause) => { + const message = cause instanceof Error ? cause.message : String(cause); + return new ServiceError({ operation, message, cause }); + }); + const deps: ProcessDependencies = { + fs, + path, + crypto, + client, + spawner, + container, + borrowCallerPath, + }; switch (creation.service) { case "rest": return catalogRecipe( diff --git a/packages/stack/src/services/Database.integration.test.ts b/packages/stack/src/services/Database.integration.test.ts index 08afa7397d..60c42e3271 100644 --- a/packages/stack/src/services/Database.integration.test.ts +++ b/packages/stack/src/services/Database.integration.test.ts @@ -18,6 +18,8 @@ import { DEFAULT_POSTGRES_ROOT_KEY } from "../Defaults.ts"; import { makeService } from "../Service.ts"; import { makeDatabase, type BackendEndpoint, type DatabaseConfig } from "./Database.ts"; import { makeDockerDatabaseRoot, runDocker } from "../../tests/docker-fixture.ts"; +import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { dockerEngineTarget } from "../../tests/engine-target.ts"; const config: DatabaseConfig = { version: "17", @@ -70,11 +72,14 @@ describe("database component", { timeout: 180_000 }, () => { ? yield* makeDockerDatabaseRoot("stack-default-key-", "default-key-test") : yield* fs.makeTempDirectoryScoped({ prefix: "stack-default-key-" }); const recipe = yield* makeDatabase({ + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, stackId: "default-key-test", instanceId: "database", root, cacheRoot: artifactCacheRoot, runtime: target.runtime, + ...(target.runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), }); const defaults: DatabaseConfig = { healthTimeoutMs: 120_000, @@ -128,6 +133,8 @@ describe("database component", { timeout: 180_000 }, () => { ).toEqual([{ decrypted_secret: "preserved-value" }]); if (target.runtime === "native") { const second = yield* makeDatabase({ + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, stackId: "default-key-test", instanceId: "database-second", root, @@ -162,6 +169,8 @@ describe("database component", { timeout: 180_000 }, () => { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-database-hba-" }); const database = yield* makeDatabase({ + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, stackId: "stack-integration", instanceId: "hba", root, @@ -221,6 +230,8 @@ describe("database component", { timeout: 180_000 }, () => { const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-database-" }); const cacheRoot = artifactCacheRoot; const first = yield* makeDatabase({ + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, stackId: "stack-integration", instanceId: "first", root, @@ -297,6 +308,8 @@ describe("database component", { timeout: 180_000 }, () => { expect(incomplete.message).toContain("major does not match"); const reopened = yield* makeDatabase({ + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, stackId: "stack-integration", instanceId: "first", root, @@ -323,6 +336,8 @@ describe("database component", { timeout: 180_000 }, () => { const secondRoot = path.join(root, "second-root"); const second = yield* makeDatabase({ + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, stackId: "stack-integration", instanceId: "second", root: secondRoot, @@ -377,11 +392,14 @@ describe("database component", { timeout: 180_000 }, () => { ); const cacheRoot = artifactCacheRoot; const database = yield* makeDatabase({ + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, stackId: "stack-integration-container", instanceId: "database", root, cacheRoot, runtime: "docker", + engineTarget: dockerEngineTarget, }); const service = yield* makeService(database.definition, { id: "database:container", @@ -432,11 +450,14 @@ describe("database component", { timeout: 180_000 }, () => { expect((yield* service.get).lifecycle).toBe("running"); yield* service.stop; const reopened = yield* makeDatabase({ + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, stackId: "stack-integration-container", instanceId: "database", root, cacheRoot, runtime: "docker", + engineTarget: dockerEngineTarget, }); const replacementPassword = Redacted.make("reopened-target-password"); const reopenedService = yield* makeService(reopened.definition, { @@ -467,12 +488,15 @@ describe("database component", { timeout: 180_000 }, () => { const stackId = "stack-compose-group"; const root = yield* makeDockerDatabaseRoot("stack-database-group-", stackId); const database = yield* makeDatabase({ + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, stackId, instanceId: "database", project: "my.app", root, cacheRoot: artifactCacheRoot, runtime: "docker", + engineTarget: dockerEngineTarget, }); const service = yield* makeService(database.definition, { id: "database:group", @@ -510,11 +534,14 @@ describe("database component", { timeout: 180_000 }, () => { const stackId = "stack-fast-shutdown"; const root = yield* makeDockerDatabaseRoot("stack-database-shutdown-", stackId); const database = yield* makeDatabase({ + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, stackId, instanceId: "database", root, cacheRoot: artifactCacheRoot, runtime: "docker", + engineTarget: dockerEngineTarget, }); const service = yield* makeService(database.definition, { id: "database:shutdown", diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index 3462e19b05..17760fd800 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -32,6 +32,7 @@ import { makeContainerRuntime, type ContainerProcess, type ContainerRuntime, + type EngineTarget, type HostGateway, } from "../runtime/Container.ts"; import { DatabaseBootstrapError, runDatabaseBootstrap } from "../runtime/DatabaseBootstrap.ts"; @@ -59,17 +60,16 @@ import { } from "../runtime/NativeProcess.ts"; import type { StackId } from "../identity/StackId.ts"; import { + acquireNativeRuntimeRoot, handOverNativePostgresFiles, openNativePostgresInstance, resolveNativePostgresUser, type PasswdEntry, } from "../runtime/postgres-user.ts"; import { EndpointIntent, serviceCreation } from "./Recipe.ts"; -import { - containerInstancePath, - ensureOwnedInstanceRoot, - removeOwnedInstanceRoot, -} from "./InstanceRoot.ts"; +import type * as Claims from "../namespace/Claims.ts"; +import * as Environment from "../namespace/Environment.ts"; +import { containerInstancePath, destroyOwnedRoot } from "../namespace/Paths.ts"; import { DEFAULT_POSTGRES_ROOT_KEY } from "../Defaults.ts"; import { instanceSnapshotsDirectory, @@ -99,7 +99,7 @@ export const DatabaseConfig = Schema.Struct({ const DatabaseReadyMarker = Schema.Struct({ version: Schema.String, - runtime: Schema.Literals(["native", "docker", "podman"]), + runtime: Schema.Literals(["native", "docker"]), profile: Schema.Literal("supabase"), }); @@ -113,7 +113,7 @@ export interface DatabaseEndpoints extends Schema.Schema.Type {} -export type DatabaseRuntime = "native" | "docker" | "podman"; +export type DatabaseRuntime = "native" | "docker"; export type BackendEndpoint = | { readonly kind: "unix"; readonly path: string; readonly port: 5432 } @@ -142,6 +142,12 @@ export interface DatabaseOptions { readonly helpers?: DockerHelperRegistry; /** Shares one host-gateway probe across this host's container runtimes. */ readonly hostGateway?: HostGateway; + /** The engine endpoint and identity the owner resolved once at startup; absent when native. */ + readonly engineTarget?: EngineTarget; + /** Journals this database's container before it is created, for the namespace's reconcile loop. */ + readonly containerClaims: Claims.ContainerClaims; + /** Journals the native socket directory this database creates under `/tmp`, outside its data root. */ + readonly directoryClaims: Claims.DirectoryClaims; } export interface DatabaseComponent { @@ -197,61 +203,66 @@ export const processExit = ( ): Effect.Effect> => sharedProcessExit(exitCode, describePostgresExit, stderr); -const reconcileContainerPassword = Effect.fn("Database.reconcileContainerPassword")(function* ( - engine: "docker" | "podman", - id: string, - password: Redacted.Redacted, - spawner: ChildProcessSpawnerService["Service"], -) { - const args = [ - "exec", - "-i", - id, - "/opt/postgres/bin/psql", - "-h", - "/tmp", - "-p", - "5432", - "-U", - "supabase_admin", - "-d", - "postgres", - "-X", - "-v", - "ON_ERROR_STOP=1", - ]; - yield* Effect.annotateCurrentSpan({ - "process.executable.name": engine, - "process.arg_count": args.length, - }); - // Each retry overwrites `process.exit_code`, so the span reports the last attempt's status. - return yield* withAttemptCount( - Effect.scoped( - Effect.gen(function* () { - const child = yield* spawner.spawn(ChildProcess.make(engine, args, { stdin: "pipe" })); - const statement = `BEGIN; SET LOCAL log_statement = 'none'; SET LOCAL log_min_error_statement = 'panic'; SET LOCAL log_min_duration_statement = -1; SET LOCAL log_min_duration_sample = -1; SET LOCAL standard_conforming_strings = on; ALTER ROLE supabase_admin PASSWORD '${Redacted.value(password).replaceAll("'", "''")}'; COMMIT;`; - const [, , , code] = yield* Effect.all( - [ - Stream.make(new TextEncoder().encode(statement)).pipe(Stream.run(child.stdin)), - child.stdout.pipe(Stream.runDrain), - child.stderr.pipe(Stream.runDrain), - child.exitCode, - ], - { concurrency: "unbounded" }, - ); - yield* Effect.annotateCurrentSpan("process.exit_code", Number(code)); - if (Number(code) !== 0) - return yield* errorFor("health", "Local database credential setup has not succeeded"); - }), - ), - (counted) => - counted.pipe( - Effect.mapError(() => errorFor("health", "Local database credential setup failed")), - Effect.retry(Schedule.spaced("250 millis")), +/** Exported for the cross-cutting pin test in {@link "../runtime/Container.integration.test.ts"}. */ +export const reconcileContainerPassword = Effect.fn("Database.reconcileContainerPassword")( + function* ( + target: EngineTarget, + id: string, + password: Redacted.Redacted, + spawner: ChildProcessSpawnerService["Service"], + ) { + const args = [ + "exec", + "-i", + id, + "/opt/postgres/bin/psql", + "-h", + "/tmp", + "-p", + "5432", + "-U", + "supabase_admin", + "-d", + "postgres", + "-X", + "-v", + "ON_ERROR_STOP=1", + ]; + yield* Effect.annotateCurrentSpan({ + "process.executable.name": target.engine, + "process.arg_count": args.length, + }); + // Each retry overwrites `process.exit_code`, so the span reports the last attempt's status. + return yield* withAttemptCount( + Effect.scoped( + Effect.gen(function* () { + const child = yield* spawner.spawn( + ChildProcess.make(target.engine, [...target.argv, ...args], { stdin: "pipe" }), + ); + const statement = `BEGIN; SET LOCAL log_statement = 'none'; SET LOCAL log_min_error_statement = 'panic'; SET LOCAL log_min_duration_statement = -1; SET LOCAL log_min_duration_sample = -1; SET LOCAL standard_conforming_strings = on; ALTER ROLE supabase_admin PASSWORD '${Redacted.value(password).replaceAll("'", "''")}'; COMMIT;`; + const [, , , code] = yield* Effect.all( + [ + Stream.make(new TextEncoder().encode(statement)).pipe(Stream.run(child.stdin)), + child.stdout.pipe(Stream.runDrain), + child.stderr.pipe(Stream.runDrain), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + yield* Effect.annotateCurrentSpan("process.exit_code", Number(code)); + if (Number(code) !== 0) + return yield* errorFor("health", "Local database credential setup has not succeeded"); + }), ), - { traced: true }, - ); -}); + (counted) => + counted.pipe( + Effect.mapError(() => errorFor("health", "Local database credential setup failed")), + Effect.retry(Schedule.spaced("250 millis")), + ), + { traced: true }, + ); + }, +); /** Idempotent readiness reconciliation, so a session also re-runs it as its probe. */ const health = Effect.fn("Database.health")(function* ( @@ -365,53 +376,6 @@ const publishLogs = publishProcessLogs; const runtimeFromContainer = (process: ContainerProcess, discard: boolean): RuntimeSession => runtimeSessionFromContainer(process, describePostgresExit, { discard }); -const databaseOwnerFileName = ".supabase-database-owner.json"; - -const ensureOwnedRoot = ( - fs: FileSystem.FileSystem, - path: Path.Path, - parentRoot: string, - stackId: string, - instanceId: string, -): Effect.Effect => - ensureOwnedInstanceRoot( - { - fs, - path, - parentRoot, - stackId, - instanceId, - ownerFileName: databaseOwnerFileName, - label: "Database root", - }, - databaseError, - ); - -const removeOwnedRoot = ( - fs: FileSystem.FileSystem, - path: Path.Path, - parentRoot: string, - stackId: string, - instanceId: string, - removeData: Effect.Effect, - /** Root entries kept with the owner marker; when empty the root itself is removed. */ - keep: ReadonlyArray = [], -): Effect.Effect => - removeOwnedInstanceRoot( - { - fs, - path, - parentRoot, - stackId, - instanceId, - ownerFileName: databaseOwnerFileName, - label: "Database root", - }, - removeData, - errorFor, - keep, - ); - const nativeProcess = ( artifact: PreparedNativeArtifact, config: DatabaseConfig, @@ -427,6 +391,7 @@ const nativeProcess = ( instanceId: string, spawner: ChildProcessSpawnerService["Service"], user: PasswdEntry | undefined, + environment: Environment.NativeEnvironment, ): Effect.Effect => spawnNativeProcess( { @@ -446,13 +411,13 @@ const nativeProcess = ( ...settings, ], env: { - ...(user === undefined ? {} : { HOME: user.home }), PGDATA: paths.dataPath, PGSODIUM_KEY_FILE: paths.rootKeyPath, POSTGRES_USER: "supabase_admin", POSTGRES_DB: "postgres", POSTGRES_PASSWORD: Redacted.value(config.databasePassword), }, + environment, gracefulStopSignal: "SIGINT", gracefulStopTimeout: "15 seconds", }, @@ -488,24 +453,31 @@ export const makeDatabase = ( const prepared = yield* Ref.make>(new Map()); if (!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/u.test(String(options.stackId))) return yield* databaseError("identity", "Invalid stack id"); - yield* ensureOwnedRoot(fs, path, options.root, String(options.stackId), options.instanceId); + // Ownership is by location: instanceRoot is owned simply by being under options.root, with no + // separate marker to establish or verify. Still created eagerly here, since the storage + // marker write below assumes the directory exists rather than creating it itself. const instanceRoot = path.join(options.root, options.instanceId); + yield* fs + .makeDirectory(instanceRoot, { recursive: true, mode: 0o700 }) + .pipe(Effect.mapError((cause) => databaseError("root", cause))); const container: ContainerRuntime | undefined = - options.runtime === "native" + options.engineTarget === undefined ? undefined : yield* makeContainerRuntime({ - engine: options.runtime, + target: options.engineTarget, root: options.root, imageMirrors: slimImageMirrors, ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), + claims: options.containerClaims, // PostgreSQL's outbound HTTP falls back across address families. awaitHostGateway: false, }); const storage: DockerDatabaseStorage | undefined = - options.runtime === "native" + options.engineTarget === undefined ? undefined : yield* makeDockerDatabaseStorage({ runtime: options.runtime, + target: options.engineTarget, stackId: String(options.stackId), instanceId: options.instanceId, ...(options.project === undefined ? {} : { project: options.project }), @@ -518,6 +490,7 @@ export const makeDatabase = ( container, spawner, ...(options.helpers === undefined ? {} : { helpers: options.helpers }), + claims: options.containerClaims, }).pipe(Effect.mapError((cause) => databaseError("storage", cause))); const snapshots = (version: string) => @@ -811,6 +784,13 @@ export const makeDatabase = ( stepDownUser === undefined ? instanceRoot : yield* asRoot(openNativePostgresInstance(stepDownUser, instanceRoot)); + // A dedicated subdirectory, never the owned root itself, so a step-down chown of it + // never touches `nativeRoot`'s own root-only ownership that the next launch verifies. + const environment = yield* Environment.confine( + fs, + path, + path.join(nativeRoot, "home"), + ).pipe(Effect.mapError((cause) => errorFor("launch", cause))); const dataPath = path.join(nativeRoot, "data"); yield* fs .makeDirectory(dataPath, { recursive: true, mode: 0o700 }) @@ -819,13 +799,34 @@ export const makeDatabase = ( yield* fs .writeFileString(rootKeyPath, Redacted.value(config.rootKey), { mode: 0o600 }) .pipe(Effect.mapError((cause) => errorFor("launch", cause))); - // PostgreSQL limits Unix socket paths to 103 bytes, independently of the user's state root. - const socketPath = yield* Effect.acquireRelease( - fs.makeTempDirectory({ directory: "/tmp", prefix: "supabase-pg-" }), - (directory) => - fs - .remove(directory, { recursive: true, force: true }) - .pipe(Effect.catch((cause) => Effect.logError(cause))), + // PostgreSQL limits Unix socket paths to 103 bytes, so this directory nests under the + // short per-uid native runtime root (acquired and validated below) rather than under + // instanceRoot. Ownership by location there keeps a foreign uid from ever creating or + // replacing it, and keeps recovery from ever having to delete outside it. + // The pathname is picked and journaled before the directory exists, so a SIGKILL + // between the claim and the create always leaves reconcile a recoverable claim. + // An existing path is never ours, so a failed exclusive create drops the claim. + const runtimeRoot = yield* asRoot(acquireNativeRuntimeRoot()); + const socketSuffix = yield* crypto.randomUUIDv4.pipe( + Effect.map((uuid) => uuid.replaceAll("-", "")), + Effect.mapError((cause) => errorFor("launch", cause)), + ); + const socketPath = path.join(runtimeRoot, `pg-${socketSuffix}`); + yield* Effect.acquireRelease( + options.directoryClaims + .claim(socketPath) + .pipe( + Effect.andThen( + fs + .makeDirectory(socketPath, { mode: 0o700 }) + .pipe(Effect.tapError(() => options.directoryClaims.unclaim(socketPath))), + ), + ), + () => + fs.remove(socketPath, { recursive: true, force: true }).pipe( + Effect.andThen(options.directoryClaims.unclaim(socketPath)), + Effect.catch((cause) => Effect.logError(cause)), + ), ).pipe( Scope.provide(context.scope), Effect.mapError((cause) => errorFor("launch", cause)), @@ -844,8 +845,10 @@ export const makeDatabase = ( rootKeyPath, socketPath, hbaPath, + runtimeRoot, bundleRoot: artifact.root, executable: artifact.executable, + environmentHome: environment.values.HOME, }), ); const process = yield* nativeProcess( @@ -858,6 +861,7 @@ export const makeDatabase = ( options.instanceId, spawner, stepDownUser, + environment, ); const selectedEndpoint: BackendEndpoint = { kind: "unix", @@ -898,15 +902,13 @@ export const makeDatabase = ( yield* Ref.set(endpoint, selectedEndpoint); yield* publishLogs(launched, logs, context.scope); const session = runtimeFromContainer(launched, config.stopGraceSeconds === 0); + const engineTarget = options.engineTarget; + if (engineTarget === undefined) + return yield* errorFor("launch", "Container runtime is unavailable"); const setup = health( selectedEndpoint, config, - reconcileContainerPassword( - options.runtime, - launched.id, - config.databasePassword, - spawner, - ), + reconcileContainerPassword(engineTarget, launched.id, config.databasePassword, spawner), { fs, instanceRoot, @@ -933,20 +935,17 @@ export const makeDatabase = ( prepare, launch, removeData: (context) => - removeOwnedRoot( + destroyOwnedRoot( fs, path, + instanceRoot, options.root, - String(options.stackId), - options.instanceId, - Effect.gen(function* () { - if (storage !== undefined) { - yield* storage.destroyData(postgresVersion(context.config.version)); - return; - } - if (container === undefined || !(yield* fs.exists(path.join(instanceRoot, "data")))) - return; - }).pipe(Effect.mapError((cause) => errorFor("destroy", cause))), + storage === undefined + ? Effect.void + : storage + .destroyData(postgresVersion(context.config.version)) + .pipe(Effect.mapError((cause) => errorFor("destroy", cause))), + errorFor, ), }; const resetData = Effect.fn("Database.resetData")(( @@ -954,22 +953,11 @@ export const makeDatabase = ( ) => { const clear: Effect.Effect = storage === undefined - ? removeOwnedRoot( - fs, - path, - options.root, - String(options.stackId), - options.instanceId, - Effect.void, - [instanceSnapshotsDirectory], - ) + ? destroyOwnedRoot(fs, path, instanceRoot, options.root, Effect.void, errorFor, [ + instanceSnapshotsDirectory, + ]) : storage.removeData(postgresVersion(context.config.version)); - return clear.pipe( - Effect.andThen( - ensureOwnedRoot(fs, path, options.root, String(options.stackId), options.instanceId), - ), - Effect.mapError((cause) => errorFor("reset", cause)), - ); + return clear.pipe(Effect.mapError((cause) => errorFor("reset", cause))); }); return { definition, diff --git a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts index a89e190335..55ac66cea7 100644 --- a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts +++ b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts @@ -3,12 +3,13 @@ import { describe, expect, it } from "@effect/vitest"; import { Crypto, Data, Effect, Exit, FileSystem, Path, Ref, Schema, Scope, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { postgresVersion, resolveArtifact } from "../Artifacts.ts"; -import { makeContainerRuntime } from "../runtime/Container.ts"; +import { makeContainerRuntime, resolveEngineTarget } from "../runtime/Container.ts"; import { makeDockerDatabaseStorage } from "../storage/DockerDatabaseStorage.ts"; import { makeDockerHelperRegistry } from "../storage/DockerHelperRegistry.ts"; import { shellQuote } from "../storage/DockerSnapshotBackend.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { makeDatabaseSnapshots, type SnapshotScope } from "./DatabaseSnapshot.ts"; +import { noContainerClaims } from "../../tests/claims.ts"; // Derived from the real catalog (not hardcoded), so a Postgres pin bump never makes this go stale. const version = postgresVersion("17"); @@ -183,7 +184,12 @@ const docker = Effect.fnUntraced(function* () { const root = yield* makeDockerDatabaseRoot("database-snapshot-contract-", stackId); const cacheRoot = path.resolve(root, "../../../cache"); const image = (yield* resolveArtifact({ service: "database", version }).pipe(Effect.orDie)).image; - const container = yield* makeContainerRuntime({ engine: "docker", root }); + const target = yield* resolveEngineTarget(spawner); + const container = yield* makeContainerRuntime({ + claims: noContainerClaims, + target, + root, + }); const helpers = yield* makeDockerHelperRegistry( `snapshot-contract-${yield* crypto.randomUUIDv4}`, ); @@ -221,8 +227,11 @@ const docker = Effect.fnUntraced(function* () { ), ); const partialCopy = rewriting(spawner, (command, args) => { - const script = args.at(-1) ?? ""; - if (command !== "docker" || args[0] !== "exec" || !script.includes("/usr/local/bin/cp")) + // Every engine invocation now carries the pinned `target.argv` prefix (e.g. `--host `) + // ahead of the real subcommand; skip past it before matching on "exec". + const unpinned = args.slice(target.argv.length); + const script = unpinned.at(-1) ?? ""; + if (command !== "docker" || unpinned[0] !== "exec" || !script.includes("/usr/local/bin/cp")) return undefined; return [ command, @@ -239,7 +248,9 @@ const docker = Effect.fnUntraced(function* () { const instanceRoot = `${root}/${name}`; yield* fs.makeDirectory(instanceRoot, { recursive: true }); const storage = yield* makeDockerDatabaseStorage({ + claims: noContainerClaims, runtime: "docker", + target, stackId, instanceId: name, instanceRoot, diff --git a/packages/stack/src/services/DatabaseSnapshot.ts b/packages/stack/src/services/DatabaseSnapshot.ts index eb335fcaa1..dfb4d6ffd0 100644 --- a/packages/stack/src/services/DatabaseSnapshot.ts +++ b/packages/stack/src/services/DatabaseSnapshot.ts @@ -17,6 +17,7 @@ import type { PlatformError } from "effect/PlatformError"; import { ChildProcessSpawner } from "effect/unstable/process"; import { postgresVersion } from "../Artifacts.ts"; import { failureMessage } from "../internal/failure-message.ts"; +import { lstatPath } from "../namespace/drivers/FileSystem.ts"; import { copyDirectory, type DirectoryCopyError } from "../storage/DirectoryCopy.ts"; import type { DatabaseRuntime } from "./Database.ts"; @@ -100,7 +101,7 @@ export interface SnapshotBackend { /** Cache-scoped entries kept besides the one just saved; instance-scoped checkpoints are never pruned. */ const retainedPrevious = 2; const format = "supabase-database-snapshot-v1" as const; -const runtimes = Schema.Literals(["native", "docker", "podman"]); +const runtimes = Schema.Literals(["native", "docker"]); const SnapshotIdentity = Schema.Struct({ format: Schema.Literal(format), version: Schema.String, @@ -411,7 +412,7 @@ const makeNativeSnapshotBackend = Effect.fnUntraced(function* (options: { fs.makeDirectory(directory, { recursive: true, mode: 0o700 }).pipe(Effect.as(proceed)), Clear: ({ directory }) => Effect.gen(function* () { - if (yield* fs.readLink(directory).pipe(Effect.isSuccess)) + if ((yield* lstatPath(directory))?.type === "SymbolicLink") return yield* errorFor("clear", `${directory} is a symbolic link`); yield* fs.makeDirectory(directory, { recursive: true, mode: 0o700 }); for (const name of yield* fs.readDirectory(directory)) diff --git a/packages/stack/src/services/Functions.integration.test.ts b/packages/stack/src/services/Functions.integration.test.ts index 36042572d6..ccc785646c 100644 --- a/packages/stack/src/services/Functions.integration.test.ts +++ b/packages/stack/src/services/Functions.integration.test.ts @@ -20,6 +20,8 @@ import { ContainerError, type ContainerRuntime } from "../runtime/Container.ts"; import { makeService } from "../Service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import * as Functions from "./Functions.ts"; +import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ stackId: "catalog-functions", @@ -32,6 +34,7 @@ const options = (root: string) => ({ const dockerOptions = (root: string) => ({ ...options(root), runtime: "docker" as const, + engineTarget: dockerEngineTarget, }); const dockerInfo = Effect.scoped( @@ -55,7 +58,12 @@ describe("service catalog", () => { const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-functions-" }); const stackId = "b".repeat(64); const instanceId = "functions-instance"; - const functionsRoot = root + "/user-functions"; + // Ownership is by location: a borrowed caller path must live outside the whole stack + // data root, so the user's functions project lives in its own, separate tree. + const callerRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "catalog-functions-caller-", + }); + const functionsRoot = callerRoot + "/user-functions"; yield* fs.makeDirectory(functionsRoot + "/hello", { recursive: true }); yield* fs.writeFileString( functionsRoot + "/hello/index.ts", @@ -91,7 +99,13 @@ describe("service catalog", () => { inspector: true, }, }, - { ...dockerOptions(root), stackId, instanceId }, + { + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, + ...dockerOptions(root), + stackId, + instanceId, + }, ); const instance = yield* makeService(recipe.definition, { id: instanceId, @@ -130,6 +144,58 @@ describe("service catalog", () => { { timeout: 120_000 }, ); + it.live("destroy survives a symlink planted where its runtime directory used to be", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "functions-symlink-" }); + const stackId = "c".repeat(64); + const instanceId = "functions-symlink-instance"; + const callerRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "functions-symlink-caller-", + }); + const functionsRoot = `${callerRoot}/user-functions`; + yield* fs.makeDirectory(functionsRoot, { recursive: true }); + yield* fs.writeFileString(`${functionsRoot}/index.ts`, "export default {};"); + const recipe = yield* makeServiceRecipe( + { + service: "functions", + config: { functionsRoot, databaseUrl: "postgres://functions-db" }, + }, + { + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, + ...options(root), + stackId, + instanceId, + }, + ); + const config = recipe.creation; + yield* recipe.definition.prepare?.(config) ?? Effect.void; + const instanceRoot = path.join(root, instanceId); + const runtimeDir = path.join(instanceRoot, "runtime"); + expect(yield* fs.exists(runtimeDir)).toBe(true); + const outside = yield* fs.makeTempDirectoryScoped({ + prefix: "functions-symlink-outside-", + }); + // The pre-F2 removeData reached for `/functions` specifically, not the whole + // runtime directory; the sentinel sits exactly there so this test actually proves that + // previously endangered data survives, not just the symlink target's top level. + const sentinel = path.join(outside, "functions", "sentinel"); + yield* fs.makeDirectory(path.join(outside, "functions"), { recursive: true }); + yield* fs.writeFileString(sentinel, "do not remove me"); + // The planted attack: the recipe's own runtime directory replaced by a symlink. + yield* fs.remove(runtimeDir, { recursive: true, force: true }); + yield* fs.symlink(outside, runtimeDir); + const scope = yield* Scope.make(); + yield* recipe.definition.removeData({ id: instanceId, config, scope }); + yield* Scope.close(scope, Exit.void); + expect(yield* fs.exists(sentinel)).toBe(true); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); + const ancestors = [ { name: "a package.json with an unreadable sibling", @@ -164,6 +230,11 @@ describe("service catalog", () => { prefix: "functions-ancestor-", }); yield* ancestor.setup(fs, root); + // Separate from the stack's own data root: this test's ancestor walk needs + // functionsRoot nested under `root`, which a borrowed caller path may not be. + const stackRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "functions-ancestor-stack-", + }); const functionsRoot = `${root}/project/supabase/functions`; yield* fs.makeDirectory(`${functionsRoot}/hello`, { recursive: true }); yield* fs.makeDirectory(`${functionsRoot}/_shared`, { recursive: true }); @@ -188,7 +259,9 @@ describe("service catalog", () => { }, }, { - ...options(root), + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, + ...options(stackRoot), stackId: "d".repeat(64), instanceId: "ancestor", cacheRoot: "/tmp/supabase-stack-artifacts", @@ -232,6 +305,9 @@ describe("service catalog", () => { directory: temporaryRoot, prefix: "functions-deno-config-", }); + const stackRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "functions-deno-config-stack-", + }); const functionsRoot = `${root}/supabase/functions`; yield* fs.makeDirectory(`${functionsRoot}/hello`, { recursive: true }); yield* fs.writeFileString( @@ -256,7 +332,9 @@ describe("service catalog", () => { }, }, { - ...options(root), + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, + ...options(stackRoot), stackId: "e".repeat(64), instanceId: "deno-config", cacheRoot: "/tmp/supabase-stack-artifacts", @@ -302,6 +380,9 @@ describe("service catalog", () => { directory: temporaryRoot, prefix: "functions-plain-deno-config-", }); + const stackRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "functions-plain-deno-config-stack-", + }); const functionsRoot = `${root}/supabase/functions`; yield* fs.makeDirectory(`${functionsRoot}/hello`, { recursive: true }); yield* fs.makeDirectory(`${functionsRoot}/_shared`, { recursive: true }); @@ -327,7 +408,9 @@ describe("service catalog", () => { }, }, { - ...options(root), + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, + ...options(stackRoot), stackId: "f".repeat(64), instanceId: "plain-deno-config", cacheRoot: "/tmp/supabase-stack-artifacts", @@ -389,6 +472,10 @@ for (const runtime of ["native", "docker"] as const) { directory: temporaryRoot, prefix: "functions-configured-", }); + const stackRoot = yield* fs.makeTempDirectoryScoped({ + directory: temporaryRoot, + prefix: "functions-configured-stack-", + }); const filesRoot = `${root}/project`; const functionsRoot = `${filesRoot}/supabase/functions`; yield* fs.makeDirectory(`${functionsRoot}/locked`, { recursive: true }); @@ -436,10 +523,13 @@ for (const runtime of ["native", "docker"] as const) { }, }, { - ...options(root), + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, + ...options(stackRoot), stackId: "c".repeat(64), instanceId: "configured", runtime, + ...(runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), cacheRoot: "/tmp/supabase-stack-artifacts", }, ); @@ -539,6 +629,8 @@ it.effect("passes POSIX project paths to a docker Functions container from a Win const recipe = yield* Functions.makeRecipe( creation, { + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, stackId: "e".repeat(64), instanceId: "windows", root: "C:\\Users\\dev\\stack", @@ -546,6 +638,7 @@ it.effect("passes POSIX project paths to a docker Functions container from a Win runtime: "docker", }, { + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), fs: yield* FileSystem.FileSystem, path: yield* Path.Path.pipe(Effect.provide(NodePath.layerWin32)), crypto: yield* Crypto.Crypto, diff --git a/packages/stack/src/services/Functions.ts b/packages/stack/src/services/Functions.ts index d3c81eb740..403dcdc6de 100644 --- a/packages/stack/src/services/Functions.ts +++ b/packages/stack/src/services/Functions.ts @@ -218,17 +218,14 @@ const makeSpec = ( }), ), ), - removeData: () => - bootstrap.cleanupAll.pipe( - Effect.mapError( - (cause) => - new ServiceError({ - operation: "destroy", - message: "Unable to remove Functions bootstrap", - cause, - }), - ), - ), + // No removeData: the bootstrap directory lives under instanceRoot, which ProcessRecipe's + // destroyOwnedRoot already removes as a single recursive delete starting from a root it has + // just confirmed is real; a selective removeData reaching into a nested path on its own could + // instead follow a symlink an attacker (or a race) planted there, deleting outside it. + callerPaths: (creation) => + creation.config.filesRoot === undefined + ? [creation.config.functionsRoot] + : [creation.config.functionsRoot, creation.config.filesRoot], }); export const makeRecipe = Effect.fn("Functions.makeRecipe")( diff --git a/packages/stack/src/services/Imgproxy.ts b/packages/stack/src/services/Imgproxy.ts index 7c497b7760..cc5dbbca72 100644 --- a/packages/stack/src/services/Imgproxy.ts +++ b/packages/stack/src/services/Imgproxy.ts @@ -35,4 +35,6 @@ export const makeSpec = (): ProcessRecipeSpec => ({ : [{ source: creation.config.filePath, target: "/mnt", readOnly: true }], ), startupCommands: [], + callerPaths: (creation) => + creation.config.filePath === undefined ? [] : [creation.config.filePath], }); diff --git a/packages/stack/src/services/InstanceRoot.integration.test.ts b/packages/stack/src/services/InstanceRoot.integration.test.ts deleted file mode 100644 index 4d8f003f3e..0000000000 --- a/packages/stack/src/services/InstanceRoot.integration.test.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Data, Effect, Exit, FileSystem, Option, Path, PlatformError } from "effect"; -import { ensureOwnedInstanceRoot, type OwnedInstanceRootParams } from "./InstanceRoot.ts"; - -class TestInstanceRootError extends Data.TaggedError("TestInstanceRootError")<{ - readonly operation: string; - readonly cause: unknown; -}> {} - -const onError = (operation: string, cause: unknown) => - new TestInstanceRootError({ operation, cause }); - -const run = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - -describe("InstanceRoot", () => { - it.live("lets concurrent first claims of a fresh root all succeed", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const parentRoot = yield* fs.makeTempDirectoryScoped({ prefix: "instance-root-race-" }); - const params: OwnedInstanceRootParams = { - fs, - path, - parentRoot, - stackId: "stack", - instanceId: "instance", - ownerFileName: ".supabase-instance-owner.json", - label: "Instance root", - }; - const results = yield* Effect.all( - Array.from({ length: 10 }, () => Effect.exit(ensureOwnedInstanceRoot(params, onError))), - { concurrency: "unbounded" }, - ); - expect(results.every(Exit.isSuccess)).toBe(true); - const marker = yield* fs.readFileString( - path.join(parentRoot, "instance", ".supabase-instance-owner.json"), - ); - expect(marker).toBe('{"stackId":"stack","instanceId":"instance"}'); - }), - ), - ); - - it.live("lets exactly one of concurrent conflicting claims of a fresh root win", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const parentRoot = yield* fs.makeTempDirectoryScoped({ prefix: "instance-root-conflict-" }); - const ownerFileName = ".supabase-instance-owner.json"; - const paramsFor = (stackId: string): OwnedInstanceRootParams => ({ - fs, - path, - parentRoot, - stackId, - instanceId: "instance", - ownerFileName, - label: "Instance root", - }); - const stackIds = Array.from({ length: 10 }, (_, i) => `stack-${i}`); - const results = yield* Effect.all( - stackIds.map((stackId) => - Effect.exit(ensureOwnedInstanceRoot(paramsFor(stackId), onError)), - ), - { concurrency: "unbounded" }, - ); - const failures = results.filter(Exit.isFailure); - expect(results.filter(Exit.isSuccess)).toHaveLength(1); - expect(failures).toHaveLength(9); - for (const failure of failures) - expect(Option.getOrUndefined(Cause.findErrorOption(failure.cause))?.cause).toBe( - "Instance root belongs to another instance", - ); - const marker = yield* fs.readFileString(path.join(parentRoot, "instance", ownerFileName)); - expect( - stackIds.map((stackId) => JSON.stringify({ stackId, instanceId: "instance" })), - ).toContain(marker); - }), - ), - ); - - it.live("ignores an orphaned staging file when claiming a fresh root", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const parentRoot = yield* fs.makeTempDirectoryScoped({ prefix: "instance-root-orphan-" }); - const ownerFileName = ".supabase-instance-owner.json"; - const root = path.join(parentRoot, "instance"); - yield* fs.makeDirectory(root, { recursive: true }); - yield* fs.writeFileString(path.join(root, `.${ownerFileName}.orphan.tmp`), "leftover"); - const params: OwnedInstanceRootParams = { - fs, - path, - parentRoot, - stackId: "stack", - instanceId: "instance", - ownerFileName, - label: "Instance root", - }; - yield* ensureOwnedInstanceRoot(params, onError); - const marker = yield* fs.readFileString(path.join(root, ownerFileName)); - expect(marker).toBe('{"stackId":"stack","instanceId":"instance"}'); - }), - ), - ); - - it.live("falls back to an exclusive create when hard links are unsupported", () => - run( - Effect.gen(function* () { - const realFs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const parentRoot = yield* realFs.makeTempDirectoryScoped({ - prefix: "instance-root-fallback-", - }); - const ownerFileName = ".supabase-instance-owner.json"; - const fs: FileSystem.FileSystem = { - ...realFs, - link: () => - Effect.fail( - PlatformError.systemError({ - _tag: "Unknown", - module: "FileSystem", - method: "link", - description: "injected unsupported-link failure", - cause: Object.assign(new Error("Operation not supported"), { code: "EPERM" }), - }), - ), - }; - const params: OwnedInstanceRootParams = { - fs, - path, - parentRoot, - stackId: "stack", - instanceId: "instance", - ownerFileName, - label: "Instance root", - }; - yield* ensureOwnedInstanceRoot(params, onError); - const root = path.join(parentRoot, "instance"); - const marker = yield* realFs.readFileString(path.join(root, ownerFileName)); - expect(marker).toBe('{"stackId":"stack","instanceId":"instance"}'); - expect(yield* realFs.readDirectory(root)).toEqual([ownerFileName]); - }), - ), - ); -}); diff --git a/packages/stack/src/services/InstanceRoot.ts b/packages/stack/src/services/InstanceRoot.ts deleted file mode 100644 index 4f547c6753..0000000000 --- a/packages/stack/src/services/InstanceRoot.ts +++ /dev/null @@ -1,155 +0,0 @@ -import { Effect, Predicate, type FileSystem, type Path } from "effect"; -import type { PlatformError } from "effect/PlatformError"; -import { randomUUID } from "node:crypto"; - -/** The container mount target for an instance-scoped directory owned on the host. */ -export const containerInstancePath = "/instance"; - -const safeInstanceIdPattern = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/u; - -/** True when `instanceId` is safe to join to a root as one path segment, without traversal. */ -const isSafeInstanceId = (instanceId: string): boolean => safeInstanceIdPattern.test(instanceId); - -/** Name a staging file for `ownerFileName`, recognizable so directory scans can ignore it. */ -const stagingFileName = (ownerFileName: string) => `.${ownerFileName}.${randomUUID()}.tmp`; - -/** True when `entry` looks like one of our own staging files for `ownerFileName`. */ -const isStagingEntry = (ownerFileName: string, entry: string): boolean => - entry.startsWith(`.${ownerFileName}.`) && entry.endsWith(".tmp"); - -/** True when `error` signals the filesystem does not support hard links. */ -const isUnsupportedLink = (error: unknown): boolean => { - if (!Predicate.hasProperty(error, "cause")) return false; - const code = - Predicate.hasProperty(error.cause, "code") && typeof error.cause.code === "string" - ? error.cause.code - : undefined; - return code === "EPERM" || code === "ENOTSUP" || code === "EOPNOTSUPP"; -}; - -export interface OwnedInstanceRootParams { - readonly fs: FileSystem.FileSystem; - readonly path: Path.Path; - /** Parent directory the instance-scoped root is joined under, as `/`. */ - readonly parentRoot: string; - readonly stackId: string; - readonly instanceId: string; - readonly ownerFileName: string; - /** Names the owned instance root in error messages, e.g. "Database root". */ - readonly label: string; -} - -/** Claims `/` for one stack+instance pair, failing if another instance already owns it. */ -export const ensureOwnedInstanceRoot = Effect.fn("InstanceRoot.ensureOwnedInstanceRoot")(( - params: OwnedInstanceRootParams, - onError: (operation: string, cause: unknown) => E, -): Effect.Effect => { - const { fs, path, parentRoot, stackId, instanceId, ownerFileName, label } = params; - const root = path.join(parentRoot, instanceId); - const ownerFile = path.join(root, ownerFileName); - const marker = JSON.stringify({ stackId, instanceId }); - const claimExistingMarker = Effect.gen(function* () { - const existing = yield* fs - .readFileString(ownerFile) - .pipe(Effect.mapError((cause) => onError("data", cause))); - if (existing !== marker) - return yield* Effect.fail(onError("data", `${label} belongs to another instance`)); - }); - const claimIfAlreadyExists = (effect: Effect.Effect) => - effect.pipe( - Effect.catchIf( - (error) => error.reason._tag === "AlreadyExists", - () => Effect.void, - ), - ); - // Staging the full marker next to `ownerFile` and hard-linking it into place keeps `wx`'s - // exclusivity without ever exposing the partial content a direct write there could risk. - const writeMarkerAtomically = Effect.suspend(() => { - const tempFile = path.join(root, stagingFileName(ownerFileName)); - return fs.writeFileString(tempFile, marker, { mode: 0o600, flag: "wx" }).pipe( - Effect.andThen(() => - fs.link(tempFile, ownerFile).pipe( - claimIfAlreadyExists, - // Filesystems without hard links (e.g. exFAT, some network shares) fall back to the - // direct exclusive create this replaced; it stays correct, just racy on those alone. - Effect.catchIf(isUnsupportedLink, () => - fs - .writeFileString(ownerFile, marker, { mode: 0o600, flag: "wx" }) - .pipe(claimIfAlreadyExists), - ), - ), - ), - Effect.ensuring(fs.remove(tempFile, { force: true }).pipe(Effect.ignore)), - Effect.mapError((cause) => onError("data", cause)), - ); - }); - return Effect.gen(function* () { - if (!isSafeInstanceId(instanceId)) - return yield* Effect.fail(onError("data", `${label} instance id is not a safe path segment`)); - yield* fs - .makeDirectory(root, { recursive: true, mode: 0o700 }) - .pipe(Effect.mapError((cause) => onError("data", cause))); - const present = yield* fs - .exists(ownerFile) - .pipe(Effect.mapError((cause) => onError("data", cause))); - if (present) return yield* claimExistingMarker; - const entries = yield* fs.readDirectory(root).pipe( - Effect.map((entries) => entries.filter((entry) => !isStagingEntry(ownerFileName, entry))), - Effect.mapError((cause) => onError("data", cause)), - ); - if (entries.length > 0) { - // A concurrent first claim may have written the marker between the `exists` and - // `readDirectory` calls above; an otherwise-empty root is still safe to compare. - if (entries.length === 1 && entries[0] === ownerFileName) return yield* claimExistingMarker; - return yield* Effect.fail(onError("data", `${label} is non-empty and unmarked`)); - } - yield* writeMarkerAtomically; - return yield* claimExistingMarker; - }); -}); - -/** Removes `/` for one stack+instance pair after running `removeData`, keeping any `keep` entries. */ -export const removeOwnedInstanceRoot = Effect.fn("InstanceRoot.removeOwnedInstanceRoot")(( - params: OwnedInstanceRootParams, - removeData: Effect.Effect, - onError: (operation: string, cause: unknown) => E, - /** Root entries kept with the owner marker; when empty the root itself is removed. */ - keep: ReadonlyArray = [], -): Effect.Effect => { - const { fs, path, parentRoot, stackId, instanceId, ownerFileName, label } = params; - const root = path.join(parentRoot, instanceId); - const ownerFile = path.join(root, ownerFileName); - const marker = JSON.stringify({ stackId, instanceId }); - return Effect.gen(function* () { - if (!isSafeInstanceId(instanceId)) - return yield* Effect.fail( - onError("destroy", `${label} instance id is not a safe path segment`), - ); - const present = yield* fs - .exists(ownerFile) - .pipe(Effect.mapError((cause) => onError("destroy", cause))); - if (!present) { - if (yield* fs.exists(root).pipe(Effect.mapError((cause) => onError("destroy", cause)))) - return yield* Effect.fail(onError("destroy", `${label} is unmarked`)); - return; - } - const existing = yield* fs - .readFileString(ownerFile) - .pipe(Effect.mapError((cause) => onError("destroy", cause))); - if (existing !== marker) - return yield* Effect.fail(onError("destroy", `${label} belongs to another instance`)); - yield* removeData; - if (keep.length === 0) - return yield* fs - .remove(root, { recursive: true, force: true }) - .pipe(Effect.mapError((cause) => onError("destroy", cause))); - const names = yield* fs - .readDirectory(root) - .pipe(Effect.mapError((cause) => onError("destroy", cause))); - for (const name of names) - if (name !== path.basename(ownerFile) && !keep.includes(name)) - yield* fs - .remove(path.join(root, name), { recursive: true, force: true }) - .pipe(Effect.mapError((cause) => onError("destroy", cause))); - }); -}); diff --git a/packages/stack/src/services/Mail.integration.test.ts b/packages/stack/src/services/Mail.integration.test.ts index f80d9a3073..e445ef4633 100644 --- a/packages/stack/src/services/Mail.integration.test.ts +++ b/packages/stack/src/services/Mail.integration.test.ts @@ -4,6 +4,8 @@ import { Effect, Exit, FileSystem, Layer, Path } from "effect"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; import { makeService } from "../Service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; +import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ stackId: "catalog-mail", @@ -11,11 +13,14 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, }); const dockerOptions = (root: string) => ({ ...options(root), runtime: "docker" as const, + engineTarget: dockerEngineTarget, }); interface MailpitMessages { diff --git a/packages/stack/src/services/Pooler.integration.test.ts b/packages/stack/src/services/Pooler.integration.test.ts index d6e0866bc4..8b8c740536 100644 --- a/packages/stack/src/services/Pooler.integration.test.ts +++ b/packages/stack/src/services/Pooler.integration.test.ts @@ -7,6 +7,8 @@ import { ProxyError } from "../Proxy.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; +import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ stackId: "catalog-pooler", @@ -14,12 +16,15 @@ const options = (root: string) => ({ root, cacheRoot: "/tmp/supabase-stack-artifacts", runtime: "native" as const, + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, }); const dockerOptions = (root: string) => ({ ...options(root), cacheRoot: `${root}/cache`, runtime: "docker" as const, + engineTarget: dockerEngineTarget, }); describe("service catalog", () => { diff --git a/packages/stack/src/services/ProcessRecipe.integration.test.ts b/packages/stack/src/services/ProcessRecipe.integration.test.ts index 9f14cc70ce..e5b41f8557 100644 --- a/packages/stack/src/services/ProcessRecipe.integration.test.ts +++ b/packages/stack/src/services/ProcessRecipe.integration.test.ts @@ -46,6 +46,7 @@ import { import type { CatalogOptions, RecipeCreation } from "./Recipe.ts"; import * as Realtime from "./Realtime.ts"; import * as Pooler from "./Pooler.ts"; +import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; type TestCreation = RecipeCreation<"rest", Record> & { readonly service: "rest"; @@ -73,6 +74,8 @@ const options: CatalogOptions = { root: "/tmp/process-recipe-test", cacheRoot: "/tmp/process-recipe-test-cache", runtime: "docker", + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, }; const spec: ProcessRecipeSpec = { @@ -188,6 +191,7 @@ describe("ProcessRecipe launch cleanup", () => { client: yield* HttpClient.HttpClient, spawner: yield* ChildProcessSpawner.ChildProcessSpawner, container, + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), } satisfies ProcessDependencies; const recipe = yield* makeProcessRecipe(creation, options, dependencies, spec); const service = yield* makeService(recipe.definition, { @@ -275,6 +279,7 @@ describe("ProcessRecipe launch cleanup", () => { client, spawner, container: undefined, + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), } satisfies ProcessDependencies; const recipe = yield* makeProcessRecipe(creation, nativeOptions, dependencies, nativeSpec); const service = yield* makeService(recipe.definition, { @@ -385,6 +390,7 @@ describe("ProcessRecipe launch cleanup", () => { client, spawner, container: undefined, + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), } satisfies ProcessDependencies; const recipe = yield* makeProcessRecipe(creation, nativeOptions, dependencies, nativeSpec); const service = yield* makeService(recipe.definition, { @@ -445,6 +451,8 @@ const realtimeService = Effect.fn(function* (container: ContainerRuntime) { const recipe = yield* makeProcessRecipe( creation, { + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, stackId: "process-recipe-test", instanceId: "instance", root: "/unused", @@ -452,6 +460,7 @@ const realtimeService = Effect.fn(function* (container: ContainerRuntime) { runtime: "docker", }, { + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), fs: yield* FileSystem.FileSystem, path: yield* Path.Path, crypto: yield* Crypto.Crypto, @@ -655,7 +664,15 @@ const nativeRestRecipe = Effect.fn(function* ( runtime: "native", platform: { os: process.platform, arch: process.arch }, }, - { fs, path, crypto, client, spawner, container: undefined }, + { + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), + fs, + path, + crypto, + client, + spawner, + container: undefined, + }, { ...spec, env: (_creation, endpoints) => @@ -917,6 +934,8 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, stackId: "process-recipe-port-race", instanceId: "instance", root, @@ -924,7 +943,15 @@ describe("process recipe startup", () => { runtime: "native", platform: { os: process.platform, arch: process.arch }, }, - { fs, path, crypto, client, spawner: interceptingSpawner, container: undefined }, + { + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), + fs, + path, + crypto, + client, + spawner: interceptingSpawner, + container: undefined, + }, Pooler.makeSpec(), ); if (recipe.definition.prepare !== undefined) yield* recipe.definition.prepare(creation); @@ -984,6 +1011,8 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, stackId: "process-recipe-port-exhaustion", instanceId: "instance", root, @@ -992,6 +1021,7 @@ describe("process recipe startup", () => { platform: { os: process.platform, arch: process.arch }, }, { + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), fs, path, crypto, @@ -1118,6 +1148,8 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, stackId: "process-recipe-deadline", instanceId: "instance", root, @@ -1125,7 +1157,15 @@ describe("process recipe startup", () => { runtime: "native", platform: { os: process.platform, arch: process.arch }, }, - { fs, path, crypto, client, spawner: deadlineSpawner, container: undefined }, + { + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), + fs, + path, + crypto, + client, + spawner: deadlineSpawner, + container: undefined, + }, Pooler.makeSpec(), ); if (recipe.definition.prepare !== undefined) yield* recipe.definition.prepare(creation); @@ -1173,6 +1213,8 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, stackId: "process-recipe-unrelated-failure", instanceId: "instance", root, @@ -1181,6 +1223,7 @@ describe("process recipe startup", () => { platform: { os: process.platform, arch: process.arch }, }, { + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), fs, path, crypto, @@ -1277,6 +1320,8 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, stackId: "process-recipe-exit-failure", instanceId: "instance", root, @@ -1284,7 +1329,15 @@ describe("process recipe startup", () => { runtime: "native", platform: { os: process.platform, arch: process.arch }, }, - { fs, path, crypto, client, spawner: failingSpawner, container: undefined }, + { + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), + fs, + path, + crypto, + client, + spawner: failingSpawner, + container: undefined, + }, Pooler.makeSpec(), ); if (recipe.definition.prepare !== undefined) yield* recipe.definition.prepare(creation); diff --git a/packages/stack/src/services/ProcessRecipe.ts b/packages/stack/src/services/ProcessRecipe.ts index 27bb425b6e..ea4433b4e3 100644 --- a/packages/stack/src/services/ProcessRecipe.ts +++ b/packages/stack/src/services/ProcessRecipe.ts @@ -22,6 +22,8 @@ import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/u import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import * as Net from "node:net"; import { prepareNativeArtifact, resolveArtifact, type ServiceKind } from "../Artifacts.ts"; +import * as Environment from "../namespace/Environment.ts"; +import { containerInstancePath, destroyOwnedRoot, type BorrowedPath } from "../namespace/Paths.ts"; import { accepts } from "../Ports.ts"; import { type ContainerError, @@ -42,11 +44,6 @@ import { runtimeSessionFromContainer, } from "../runtime/Session.ts"; import { ServiceError, ServiceLaunchError, type RuntimeSession } from "../Service.ts"; -import { - containerInstancePath, - ensureOwnedInstanceRoot, - removeOwnedInstanceRoot, -} from "./InstanceRoot.ts"; import { type CatalogLog, CatalogError, @@ -106,6 +103,11 @@ export interface ProcessRecipeSpec Effect.Effect; /** Claims an owned instance directory, mounted at `/instance` in containers. */ readonly instanceDirectory?: boolean; + /** + * Every caller-supplied path this recipe reads or mounts, validated against the stack's data + * root before `prepare` and launch run (see `namespace/Paths.borrow`). + */ + readonly callerPaths?: (creation: C) => ReadonlyArray; } export interface ResolvedStartupCommand { @@ -162,6 +164,8 @@ export interface ProcessDependencies { readonly client: HttpClient.HttpClient; readonly spawner: ChildProcessSpawnerService["Service"]; readonly container: ContainerRuntime | undefined; + /** Validates a caller-supplied path against the stack's data root; see `namespace/Paths.borrow`. */ + readonly borrowCallerPath: (candidate: string) => Effect.Effect; } const serviceError = mapToServiceError; @@ -424,23 +428,26 @@ export const makeProcessRecipe = const preparedRoot = yield* Ref.make(undefined); const endpoints = yield* Ref.make>(new Map()); const logs = yield* PubSub.sliding(256); + // Ownership is by location: instanceRoot is owned simply by being under options.root, created + // on demand by whatever first writes under it (no separate marker to establish or verify). const instanceRoot = deps.path.join(options.root, options.instanceId); - const ownedInstanceRoot = { - fs: deps.fs, - path: deps.path, - parentRoot: options.root, - stackId: options.stackId, - instanceId: options.instanceId, - ownerFileName: ".supabase-instance-owner.json", - label: "Instance root", - }; const nativeInstanceDir = spec.instanceDirectory === true ? instanceRoot : undefined; const containerInstanceDir = spec.instanceDirectory === true ? containerInstancePath : undefined; + // A dedicated, fully namespace-owned subdirectory for native confinement, so it never mixes + // with the recipe's own owned files. + const environmentRoot = deps.path.join(instanceRoot, ".supabase-environment"); const prepare = Effect.fn("ProcessRecipe.prepare")(function* (candidate: C) { + if (!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/u.test(options.instanceId)) + return yield* serviceError("identity", "Instance id is not a safe path segment"); if (spec.instanceDirectory === true) - yield* ensureOwnedInstanceRoot(ownedInstanceRoot, serviceError); + yield* deps.fs + .makeDirectory(instanceRoot, { recursive: true, mode: 0o700 }) + .pipe(Effect.mapError((cause) => serviceError("prepare", cause))); + if (spec.callerPaths !== undefined) + for (const candidatePath of spec.callerPaths(candidate)) + yield* deps.borrowCallerPath(candidatePath); if (spec.prepare !== undefined) yield* spec.prepare(candidate); const resolved = yield* resolveArtifact({ service: candidate.service, @@ -485,6 +492,9 @@ export const makeProcessRecipe = return yield* serviceError("launch", "Artifact was not prepared"); if (artifactRoot === undefined) return yield* serviceError("launch", "Artifact root was not prepared"); + const environment = yield* Environment.confine(deps.fs, deps.path, environmentRoot).pipe( + Effect.mapError((cause) => serviceError("launch", cause)), + ); const reserveEndpoints = Effect.fn("ProcessRecipe.reserveEndpoints")(function* ( parent: Scope.Closeable, ) { @@ -537,6 +547,7 @@ export const makeProcessRecipe = executable: `${artifactRoot}/bin/${command.executable}`, args: command.args, env: command.env, + environment, cwd: artifactRoot, }, defaultNativeProcessLauncher(), @@ -584,6 +595,7 @@ export const makeProcessRecipe = executable, args, env, + environment, gracefulStopSignal: "SIGTERM", gracefulStopTimeout: "5 seconds", }, @@ -874,14 +886,16 @@ export const makeProcessRecipe = definition: { prepare, launch, + // instanceRoot (environmentRoot's parent, and the recipe's own owned files) is removed as + // one recursive delete; a recipe with nothing on the host filesystem simply finds it absent. removeData: (context) => - (spec.instanceDirectory === true - ? removeOwnedInstanceRoot( - ownedInstanceRoot, - spec.removeData?.(context.config) ?? Effect.void, - serviceError, - ) - : (spec.removeData?.(context.config) ?? Effect.void) + destroyOwnedRoot( + deps.fs, + deps.path, + instanceRoot, + options.root, + spec.removeData?.(context.config) ?? Effect.void, + serviceError, ).pipe(Effect.andThen(Ref.set(endpoints, new Map()))), }, endpoints, diff --git a/packages/stack/src/services/Realtime.integration.test.ts b/packages/stack/src/services/Realtime.integration.test.ts index 3d495a5ef9..bc7468628c 100644 --- a/packages/stack/src/services/Realtime.integration.test.ts +++ b/packages/stack/src/services/Realtime.integration.test.ts @@ -6,6 +6,8 @@ import { makeService } from "../Service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; +import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ stackId: "catalog-realtime", @@ -13,11 +15,14 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, }); const dockerOptions = (root: string) => ({ ...options(root), runtime: "docker" as const, + engineTarget: dockerEngineTarget, }); describe("service catalog", () => { diff --git a/packages/stack/src/services/Recipe.ts b/packages/stack/src/services/Recipe.ts index 940e1f5f38..7479bf2533 100644 --- a/packages/stack/src/services/Recipe.ts +++ b/packages/stack/src/services/Recipe.ts @@ -3,10 +3,11 @@ import type { Stream } from "effect"; import type { Effect, Ref } from "effect"; import type { ServiceKind } from "../Artifacts.ts"; import type { ServiceDefinition } from "../Service.ts"; -import type { HostGateway } from "../runtime/Container.ts"; +import type { ContainerClaims, DirectoryClaims } from "../namespace/Claims.ts"; +import type { EngineTarget, HostGateway } from "../runtime/Container.ts"; import type { DockerHelperRegistry } from "../storage/DockerHelperRegistry.ts"; -type CatalogRuntime = "native" | "docker" | "podman"; +type CatalogRuntime = "native" | "docker"; export class CatalogError extends Data.TaggedError("CatalogError")<{ readonly operation: string; @@ -56,6 +57,12 @@ export interface CatalogOptions { readonly helpers?: DockerHelperRegistry; /** Shares one host-gateway probe across this host's container runtimes. */ readonly hostGateway?: HostGateway; + /** The engine endpoint and identity the owner resolved once at startup; absent when native. */ + readonly engineTarget?: EngineTarget; + /** Journals this service's containers before they are created, for the namespace's reconcile loop. */ + readonly containerClaims: ContainerClaims; + /** Journals directories a service creates outside its data root (for example under `/tmp`). */ + readonly directoryClaims: DirectoryClaims; } export interface CatalogLog { diff --git a/packages/stack/src/services/Rest.integration.test.ts b/packages/stack/src/services/Rest.integration.test.ts index 779a79e542..fbb5e9f2de 100644 --- a/packages/stack/src/services/Rest.integration.test.ts +++ b/packages/stack/src/services/Rest.integration.test.ts @@ -11,6 +11,8 @@ import { makeService } from "../Service.ts"; import { ProxyError } from "../Proxy.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; +import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { dockerEngineTarget } from "../../tests/engine-target.ts"; const makeTestState = (root: string) => Layer.build(StackNamespace.layer({ root })).pipe( @@ -34,11 +36,14 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, }); const dockerOptions = (root: string) => ({ ...options(root), runtime: "docker" as const, + engineTarget: dockerEngineTarget, }); describe("service catalog", () => { diff --git a/packages/stack/src/services/Storage.ts b/packages/stack/src/services/Storage.ts index 2018dfad64..c91e78d3b1 100644 --- a/packages/stack/src/services/Storage.ts +++ b/packages/stack/src/services/Storage.ts @@ -124,4 +124,5 @@ export const makeSpec = (): ProcessRecipeSpec => ({ mounts: (creation, _context) => Effect.succeed([{ source: creation.config.filePath, target: "/mnt", readOnly: false }]), startupCommands: [initializationCommand], + callerPaths: (creation) => [creation.config.filePath], }); diff --git a/packages/stack/src/services/StorageGateway.integration.test.ts b/packages/stack/src/services/StorageGateway.integration.test.ts index ed308311eb..daa43a4e8c 100644 --- a/packages/stack/src/services/StorageGateway.integration.test.ts +++ b/packages/stack/src/services/StorageGateway.integration.test.ts @@ -12,6 +12,7 @@ import * as StackNamespace from "../StackNamespace.ts"; import type { SavedStack } from "../StackNamespace.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { ownerFor } from "../../tests/owner-rpc.ts"; +import { dockerEngineTarget } from "../../tests/engine-target.ts"; const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; const jwtSecret = "storage-gateway-secret-with-at-least-32-chars"; @@ -31,10 +32,15 @@ const layout = Effect.fnUntraced(function* (runtime: SavedStack["runtime"], stac const dataRoot = yield* makeDockerDatabaseRoot("storage-gateway-docker-", stackId).pipe( Effect.flatMap(fs.realPath), ); + // Ownership is by location: Storage's filePath is a caller path and must live outside the + // stack's data root, not merely outside the service's own instance root. + const storageRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "storage-gateway-docker-caller-", + }); return { stateRoot: path.dirname(path.dirname(dataRoot)), dataRoot, - storageRoot: `${dataRoot}/storage`, + storageRoot, }; }); @@ -59,7 +65,13 @@ const serveStorage = Effect.fnUntraced(function* (runtime: SavedStack["runtime"] StackNamespace.Service, ); yield* state.save(saved); - const owner = yield* ownerFor({ saved, state, root: dataRoot, cacheRoot }); + const owner = yield* ownerFor({ + saved, + state, + root: dataRoot, + cacheRoot, + ...(runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), + }); yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); const created = yield* owner.rpc.supabaseComposition({ services: [ diff --git a/packages/stack/src/services/Studio.integration.test.ts b/packages/stack/src/services/Studio.integration.test.ts new file mode 100644 index 0000000000..2bfb4e945d --- /dev/null +++ b/packages/stack/src/services/Studio.integration.test.ts @@ -0,0 +1,43 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Layer } from "effect"; +import { makeService } from "../Service.ts"; +import { makeServiceRecipe } from "./Catalog.ts"; +import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; + +const options = (root: string) => ({ + stackId: "catalog-studio", + instanceId: "studio", + root, + cacheRoot: `${root}/cache`, + runtime: "native" as const, + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, +}); + +describe("studio recipe", () => { + it.live( + "rejects a functionsRoot that resolves inside a sibling instance under the same data root", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-studio-reject-" }); + // Ownership is by location: Studio's functionsRoot is a caller path and must live + // outside the stack's data root, not merely outside this instance's own instance root. + const siblingInstance = `${root}/functions-instance`; + yield* fs.makeDirectory(siblingInstance, { recursive: true }); + const recipe = yield* makeServiceRecipe( + { service: "studio", config: { functionsRoot: siblingInstance } }, + options(root), + ); + const studio = yield* makeService(recipe.definition, { + id: "studio", + config: recipe.creation, + }); + const failure = yield* studio.start.pipe(Effect.flip); + expect(failure.message).toContain("resolves inside the owned data root"); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); +}); diff --git a/packages/stack/src/services/Studio.ts b/packages/stack/src/services/Studio.ts index 133e1f24ea..30a57e44e9 100644 --- a/packages/stack/src/services/Studio.ts +++ b/packages/stack/src/services/Studio.ts @@ -129,4 +129,8 @@ export const makeSpec = (): ProcessRecipeSpec => ({ ]), ]), startupCommands: [], + callerPaths: (creation) => [ + ...(creation.config.functionsRoot === undefined ? [] : [creation.config.functionsRoot]), + ...(creation.config.snippetsRoot === undefined ? [] : [creation.config.snippetsRoot]), + ], }); diff --git a/packages/stack/src/services/Vector.integration.test.ts b/packages/stack/src/services/Vector.integration.test.ts index 2eba9f07ec..4250a8b401 100644 --- a/packages/stack/src/services/Vector.integration.test.ts +++ b/packages/stack/src/services/Vector.integration.test.ts @@ -5,6 +5,8 @@ import { Effect, FileSystem, Layer } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { makeService } from "../Service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; +import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string, runtime: "docker" | "native") => ({ stackId: "catalog-test", @@ -12,6 +14,9 @@ const options = (root: string, runtime: "docker" | "native") => ({ root, cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, runtime, + ...(runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, }); describe("vector recipe", () => { @@ -45,7 +50,7 @@ describe("vector recipe", () => { HttpClientRequest.get(`http://${endpoint.host}:${endpoint.port}/health`), ); expect(response.status).toBe(200); - yield* fs.makeDirectory(`${root}/vector/runtime/vector/.vector-write-interrupted`); + yield* fs.makeDirectory(`${root}/vector/runtime/vector/.vector-api.yaml-interrupted`); yield* vector.destroy; expect(yield* fs.exists(`${root}/vector`)).toBe(false); }), @@ -55,14 +60,18 @@ describe("vector recipe", () => { } it.live( - "keeps a caller pipeline stored beside the recipe config on destroy", + "leaves a caller pipeline stored outside the owned instance root on destroy", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-vector-caller-" }); - const pipeline = `${root}/vector/runtime/vector/pipeline.yaml`; - yield* fs.makeDirectory(`${root}/vector/runtime/vector`, { recursive: true }); + // Ownership is by location: a borrowed caller config must live outside the stack's + // entire data root, not merely outside this instance's own instanceRoot. + const external = yield* fs.makeTempDirectoryScoped({ + prefix: "catalog-vector-caller-external-", + }); + const pipeline = `${external}/pipeline.yaml`; yield* fs.writeFileString( pipeline, "sources:\n s:\n type: internal_logs\nsinks:\n d:\n type: blackhole\n inputs: [s]\n", @@ -83,7 +92,7 @@ describe("vector recipe", () => { yield* vector.ready; yield* vector.destroy; expect(yield* fs.exists(pipeline)).toBe(true); - expect(yield* fs.exists(`${root}/vector/runtime/vector/vector-api.yaml`)).toBe(false); + expect(yield* fs.exists(`${root}/vector`)).toBe(false); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 120_000 }, @@ -115,7 +124,9 @@ describe("vector recipe", () => { return yield* vector.start.pipe(Effect.flip); }); for (const configPath of [owned, `${root}/alias.yaml`]) - expect((yield* start(configPath)).message).toContain("stack-owned Vector config file"); + expect((yield* start(configPath)).message).toContain( + "resolves inside the owned data root", + ); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); diff --git a/packages/stack/src/services/Vector.ts b/packages/stack/src/services/Vector.ts index ec830a7c99..2c14d71443 100644 --- a/packages/stack/src/services/Vector.ts +++ b/packages/stack/src/services/Vector.ts @@ -1,5 +1,6 @@ import { Effect, type FileSystem, type Path, Schema } from "effect"; import { ServiceError } from "../Service.ts"; +import { writeOwnedFile } from "../namespace/Paths.ts"; import { type CatalogOptions, EndpointIntent, serviceCreation } from "./Recipe.ts"; import { requiredInput } from "./ServiceConfig.ts"; import { @@ -58,34 +59,27 @@ const defaultPipelineConfig = [ const containerPipelinePath = "/etc/vector/vector.yaml"; const containerApiPath = "/etc/supabase/vector-api.yaml"; -const temporaryPrefix = ".vector-write-"; -const writeAtomically = Effect.fn("Vector.writeAtomically")( - function* (fs: FileSystem.FileSystem, path: Path.Path, target: string, content: string) { - const directory = path.dirname(target); - yield* fs.makeDirectory(directory, { recursive: true, mode: 0o700 }); - yield* Effect.acquireUseRelease( - fs.makeTempDirectory({ directory, prefix: temporaryPrefix }), - (temporaryDirectory) => - Effect.gen(function* () { - const temporary = path.join(temporaryDirectory, path.basename(target)); - yield* fs.writeFileString(temporary, content, { mode: 0o644 }); - yield* fs.rename(temporary, target); - }), - (temporaryDirectory) => - fs - .remove(temporaryDirectory, { recursive: true, force: true }) - .pipe(Effect.catchTag("PlatformError", () => Effect.void)), - ); - }, - Effect.mapError( - (cause) => - new ServiceError({ operation: "prepare", message: "Unable to write Vector config", cause }), - ), -); +const writeAtomically = ( + fs: FileSystem.FileSystem, + path: Path.Path, + target: string, + content: string, +) => + writeOwnedFile( + fs, + path, + target, + content, + (operation, cause) => + new ServiceError({ + operation, + message: "Unable to write Vector config", + cause, + }), + ); const makeSpec = ( - instanceId: string, instanceRoot: string, fs: FileSystem.FileSystem, path: Path.Path, @@ -97,10 +91,6 @@ const makeSpec = ( /** What Vector actually loads: a caller's file is rendered to a recipe-owned copy first. */ const resolvedPipelinePath = (creation: Creation) => creation.config.configPath === undefined ? defaultPipelinePath : renderedPipelinePath; - const ownedInstance = (operation: string) => - /^[a-zA-Z0-9_-]+$/u.test(instanceId) - ? Effect.void - : Effect.fail(new ServiceError({ operation, message: "Invalid Vector instance identity" })); return { service: "vector", executable: "bin/vector", @@ -124,6 +114,7 @@ const makeSpec = ( : `${context.container ? "0.0.0.0" : "127.0.0.1"}:${http.port}`; yield* writeAtomically(fs, path, apiConfigPath, apiConfigFor(address)); if (creation.config.configPath !== undefined) { + // Validated by `callerPaths` before prepare ran: read-only, no delete authority over it. const source = yield* fs.readFileString(creation.config.configPath).pipe( Effect.mapError( (cause) => @@ -152,80 +143,18 @@ const makeSpec = ( startupCommands: [], prepare: (creation) => Effect.gen(function* () { - yield* ownedInstance("prepare"); const callerPath = creation.config.configPath; - if (callerPath !== undefined) { - const canonical = (file: string) => - fs - .exists(file) - .pipe( - Effect.flatMap((exists) => - exists ? fs.realPath(file) : Effect.succeed(path.resolve(file)), - ), - ); - const caller = yield* fs.realPath(callerPath).pipe( - Effect.mapError( - (cause) => - new ServiceError({ - operation: "prepare", - message: "Unable to read Vector configPath", - cause, - }), - ), - ); - const owned = yield* Effect.all([ - canonical(apiConfigPath), - canonical(defaultPipelinePath), - canonical(renderedPipelinePath), - ]).pipe( - Effect.mapError( - (cause) => - new ServiceError({ - operation: "prepare", - message: "Unable to resolve Vector config paths", - cause, - }), - ), - ); - if (owned.includes(caller)) - return yield* new ServiceError({ - operation: "prepare", - message: "Vector configPath must not point at a stack-owned Vector config file", - }); - } // A safe placeholder until `args` writes the real address — the listening port isn't // known until endpoints are reserved for an actual launch, which happens after `prepare`. yield* writeAtomically(fs, path, apiConfigPath, apiConfigFor(undefined)); if (callerPath === undefined) yield* writeAtomically(fs, path, defaultPipelinePath, defaultPipelineConfig); }), - // A caller configPath may live anywhere under the instance root, so only recipe files and empty directories go. - removeData: () => - Effect.gen(function* () { - yield* ownedInstance("destroy"); - yield* fs.remove(apiConfigPath, { force: true }); - yield* fs.remove(defaultPipelinePath, { force: true }); - yield* fs.remove(renderedPipelinePath, { force: true }); - if (yield* fs.exists(configRoot)) - for (const entry of yield* fs.readDirectory(configRoot)) - if (entry.startsWith(temporaryPrefix)) - yield* fs.remove(path.join(configRoot, entry), { recursive: true, force: true }); - for (const directory of [configRoot, path.dirname(configRoot), instanceRoot]) { - if (!(yield* fs.exists(directory))) continue; - if ((yield* fs.readDirectory(directory)).length > 0) return; - yield* fs.remove(directory, { recursive: true }); - } - }).pipe( - Effect.mapError((cause) => - cause instanceof ServiceError - ? cause - : new ServiceError({ - operation: "destroy", - message: "Unable to remove Vector config", - cause, - }), - ), - ), + // No removeData: configRoot lives under instanceRoot, which ProcessRecipe's destroyOwnedRoot + // already removes recursively; a caller's borrowed config sits outside it by construction, + // validated by callerPaths against the whole stack data root, not just this instance root. + callerPaths: (creation) => + creation.config.configPath === undefined ? [] : [creation.config.configPath], }; }; @@ -235,11 +164,6 @@ export const makeRecipe = Effect.fn("Vector.makeRecipe")( creation, options, deps, - makeSpec( - options.instanceId, - deps.path.join(options.root, options.instanceId), - deps.fs, - deps.path, - ), + makeSpec(deps.path.join(options.root, options.instanceId), deps.fs, deps.path), ), ); diff --git a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts index 16619a0b77..718c066403 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts @@ -2,6 +2,7 @@ import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { ConfigProvider, + Context, Crypto, Data, Deferred, @@ -19,14 +20,20 @@ import { Stream, } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { fileURLToPath } from "node:url"; import { postgresVersion, resolveArtifact } from "../Artifacts.ts"; -import { makeContainerRuntime } from "../runtime/Container.ts"; +import * as Claims from "../namespace/Claims.ts"; +import { makeContainerRuntime, type EngineTarget } from "../runtime/Container.ts"; import { makeDatabaseSnapshots } from "../services/DatabaseSnapshot.ts"; +import * as StackNamespace from "../StackNamespace.ts"; import { makeDockerDatabaseStorage } from "./DockerDatabaseStorage.ts"; import { makeDockerHelperRegistry } from "./DockerHelperRegistry.ts"; import { removeTestRunVolumes } from "../../tests/docker-volume-run.ts"; import type { DockerHelperRegistry } from "./DockerHelperRegistry.ts"; +import { noContainerClaims } from "../../tests/claims.ts"; +// An unpinned target: these tests exercise storage marker/volume logic, not endpoint pinning. +const dockerTarget: EngineTarget = { engine: "docker", argv: [], daemonId: "test-daemon-id" }; const postgresImage = (version: string) => resolveArtifact({ service: "database", version: postgresVersion(version) }).pipe( Effect.map(({ image }) => image), @@ -43,9 +50,19 @@ const Marker = Schema.Struct({ class DockerTestError extends Data.TaggedError("DockerTestError")<{ readonly message: string; }> {} +class PublishBarrierError extends Data.TaggedError("PublishBarrierError")<{ + readonly message: string; +}> {} + +const dockerStoragePublishLoopFixture = fileURLToPath( + new URL("../../tests/docker-storage-publish-loop-fixture.ts", import.meta.url), +); const helperMirror = "registry.test/supabase/postgres:17"; -const fakeHelperEngine = () => { +const fakeHelperEngine = ( + onRun: () => Effect.Effect = () => Effect.void, + { volumePresent = false }: { readonly volumePresent?: boolean } = {}, +) => { const commands: string[][] = []; const local = new Set(); const handle = (exitCode: number, stdout = "", stderr = "") => @@ -62,29 +79,38 @@ const fakeHelperEngine = () => { getOutputFd: () => Stream.empty, unref: Effect.succeed(Effect.void), }); - const spawner = ChildProcessSpawner.make((command) => { - if (!ChildProcess.isStandardCommand(command)) return Effect.die("unexpected piped command"); - const args = [...command.args]; - commands.push(args); - if (args[0] === "image") - return Effect.succeed(handle(0, local.has(args.at(-1) ?? "") ? "sha256:1" : "")); - if (args[0] === "pull") { - if (args.at(-1) !== helperMirror) - return Effect.succeed(handle(1, "", `denied: ${args.at(-1)}`)); - local.add(helperMirror); - return Effect.succeed(handle(0)); - } - if (args[0] === "inspect") return Effect.succeed(handle(1, "", "no such container")); - if (args[0] === "run") { - const shellIndex = args.indexOf("/bin/sh"); - const image = args[shellIndex - 1]; - return image !== undefined && local.has(image) - ? Effect.succeed(handle(0, "abcdef0123456789")) - : Effect.succeed(handle(1, "", `Unable to find image '${image ?? ""}' locally`)); - } - if (args[0] === "exec" || args[0] === "rm") return Effect.succeed(handle(0, "done")); - return Effect.succeed(handle(1, "", `unexpected engine command: ${args[0] ?? ""}`)); - }); + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if (!ChildProcess.isStandardCommand(command)) + return yield* Effect.die("unexpected piped command"); + const args = [...command.args]; + commands.push(args); + if (args[0] === "info") return handle(0, "fake-daemon-id"); + if (args[0] === "version") return handle(0, "27.0.0|27.0.0"); + if (args[0] === "volume" && args[1] === "inspect") + return volumePresent ? handle(0) : handle(1, "", "no such volume"); + if (args[0] === "volume" && args[1] === "create") return handle(0); + if (args[0] === "image") return handle(0, local.has(args.at(-1) ?? "") ? "sha256:1" : ""); + if (args[0] === "pull") { + if (args.at(-1) !== helperMirror) return handle(1, "", `denied: ${args.at(-1)}`); + local.add(helperMirror); + return handle(0); + } + if (args[0] === "inspect") return handle(1, "", "no such container"); + if (args[0] === "run") { + const shellIndex = args.indexOf("/bin/sh"); + const image = args[shellIndex - 1]; + if (image === undefined || !local.has(image)) + return handle(1, "", `Unable to find image '${image ?? ""}' locally`); + // Fires once the create command the claim must precede is observed, so the claim's + // presence at this exact point is a deterministic fact, not a timing guess. + yield* onRun(); + return handle(0, "abcdef0123456789"); + } + if (args[0] === "exec" || args[0] === "rm") return handle(0, "done"); + return handle(1, "", `unexpected engine command: ${args[0] ?? ""}`); + }), + ); return { commands, layer: Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner) }; }; @@ -141,6 +167,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { if (!ChildProcess.isStandardCommand(command)) return yield* Effect.die("Unexpected command"); const creating = command.args[0] === "run"; + const identity = command.args[0] === "info"; if (creating) yield* Deferred.succeed(started, undefined); if (command.args[0] === "rm") { yield* Ref.update(removals, (count) => count + 1); @@ -157,7 +184,9 @@ describe("Docker database storage", { timeout: 120_000 }, () => { stdin: Sink.drain, stdout: creating ? Stream.succeed(new TextEncoder().encode("a".repeat(64))) - : Stream.empty, + : identity + ? Stream.succeed(new TextEncoder().encode('{"host":{"id":"fake-daemon-id"}}')) + : Stream.empty, stderr: Stream.empty, all: Stream.empty, getInputFd: () => Sink.drain, @@ -167,7 +196,9 @@ describe("Docker database storage", { timeout: 120_000 }, () => { }), ); const storage = yield* makeDockerDatabaseStorage({ - runtime: "podman", + claims: noContainerClaims, + runtime: "docker", + target: dockerTarget, stackId: "helper-create-cancel", instanceId: "database", instanceRoot: root, @@ -210,13 +241,16 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const instanceRoot = path.join(storageRoot, "database"); yield* fs.makeDirectory(instanceRoot, { recursive: true }); const container = yield* makeContainerRuntime({ - engine: "podman", + claims: noContainerClaims, + target: dockerTarget, root, imageMirrors: () => [helperMirror], }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const storage = yield* makeDockerDatabaseStorage({ - runtime: "podman", + claims: noContainerClaims, + runtime: "docker", + target: dockerTarget, stackId: "storage-helper-mirror", instanceId: "database", instanceRoot, @@ -242,8 +276,86 @@ describe("Docker database storage", { timeout: 120_000 }, () => { ).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); }); + it.live("claims a helper container before creating it, and unclaims it after removal", () => { + // Assigned once `state` exists below; the mock engine calls it synchronously at the exact + // moment the "run" command (the one the claim must precede) fires. + let onCreate: () => Effect.Effect = () => Effect.void; + const engine = fakeHelperEngine(() => onCreate()); + return Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "storage-helper-claims-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "database"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root: path.join(root, "state", "stack") })), + StackNamespace.Service, + ); + const stackId = "storage-helper-claims"; + const claims = Claims.forStack(state, stackId).containers; + const claimedAtCreate = yield* Deferred.make>(); + onCreate = () => + state.readClaims(stackId).pipe( + Effect.flatMap((current) => Deferred.succeed(claimedAtCreate, current)), + Effect.asVoid, + Effect.orDie, + ); + // Scoped on its own: the helper is a reusable resource unclaimed only when the + // storage's own scope closes (its removeHelper finalizer), not merely after one + // operation, so that close must happen before the unclaim assertion below. + yield* Effect.scoped( + Effect.gen(function* () { + const container = yield* makeContainerRuntime({ + claims, + target: dockerTarget, + root, + imageMirrors: () => [helperMirror], + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const storage = yield* makeDockerDatabaseStorage({ + claims, + runtime: "docker", + target: dockerTarget, + stackId, + instanceId: "database", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + + yield* storage.prepare("17"); + yield* storage.removeData("17"); + }), + ); + + // No sleeps or polling: the deferred is only ever resolved from inside the mock engine, + // synchronously at the claim-then-create boundary. + const atCreate = yield* Deferred.await(claimedAtCreate); + expect(atCreate).toHaveLength(1); + expect(atCreate[0]).toMatchObject({ kind: "container" }); + const runCommand = engine.commands.find((args) => args[0] === "run"); + const nameIndex = runCommand?.indexOf("--name") ?? -1; + const namedByEngine = nameIndex === -1 ? undefined : runCommand?.[nameIndex + 1]; + expect(namedByEngine).toBeDefined(); + // The journaled identity is exactly the one the engine was told to create, not merely + // some container identity: a mismatch would let reconcile remove the wrong resource. + expect(atCreate[0]?.id).toBe(namedByEngine); + expect(yield* state.readClaims(stackId)).toEqual([]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); + it.live("starts a shared volume helper with the mirror image selected during preparation", () => { - const engine = fakeHelperEngine(); + const engine = fakeHelperEngine(undefined, { volumePresent: true }); return Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -264,11 +376,22 @@ describe("Docker database storage", { timeout: 120_000 }, () => { ) .join("") .slice(0, 32)}`; + // Matches the volume name the engine target's own identity resolves, the same way + // DockerDatabaseStorage derives it: a stable digest of the state root and daemon id. + const statePath = yield* fs.realPath(path.join(root, "state")); + const stateHash = yield* crypto.digest( + "SHA-256", + new TextEncoder().encode(`${statePath}\0${dockerTarget.daemonId}`), + ); + const stateDigest = Array.from(stateHash, (byte) => byte.toString(16).padStart(2, "0")) + .join("") + .slice(0, 32); const marker = yield* Schema.encodeEffect(Schema.fromJsonString(Marker))({ backend: "docker", - volume: "database-volume", + volume: `supabase-db-${stateDigest}`, namespace: `instance-${stackId}-${instanceId}`, cacheNamespace, + daemonId: dockerTarget.daemonId, initialized: false, }); yield* fs.writeFileString( @@ -276,14 +399,17 @@ describe("Docker database storage", { timeout: 120_000 }, () => { marker, ); const container = yield* makeContainerRuntime({ - engine: "podman", + claims: noContainerClaims, + target: dockerTarget, root, imageMirrors: () => [helperMirror], }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const helpers = yield* makeDockerHelperRegistry("mirror-test"); const storage = yield* makeDockerDatabaseStorage({ - runtime: "podman", + claims: noContainerClaims, + runtime: "docker", + target: dockerTarget, stackId, instanceId, instanceRoot, @@ -324,12 +450,18 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* fs.makeDirectory(sourceRoot, { recursive: true }); yield* fs.makeDirectory(targetRoot, { recursive: true }); yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ engine: "docker", root }); + const container = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root, + }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const stackId = `storage-pg15-${yield* crypto.randomUUIDv4}`; const makeStorage = (instanceId: string, instanceRoot: string) => makeDockerDatabaseStorage({ + claims: noContainerClaims, runtime: "docker", + target: dockerTarget, stackId, instanceId, instanceRoot, @@ -427,7 +559,11 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* fs.makeDirectory(sourceRoot, { recursive: true }); yield* fs.makeDirectory(targetRoot, { recursive: true }); yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ engine: "docker", root }); + const container = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root, + }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const makeStorageAt = ( instanceId: string, @@ -436,7 +572,9 @@ describe("Docker database storage", { timeout: 120_000 }, () => { cacheRootValue: string, ) => makeDockerDatabaseStorage({ + claims: noContainerClaims, runtime: "docker", + target: dockerTarget, stackId: "storage-test", instanceId, instanceRoot, @@ -620,14 +758,20 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const cacheRoot = path.join(root, "cache"); const instanceRoot = path.join(storageRoot, "database"); yield* fs.makeDirectory(instanceRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ engine: "docker", root }); + const container = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root, + }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const stackId = `storage-test-run-${yield* crypto.randomUUIDv4}`; const testRunId = `storage-test-run-${(yield* crypto.randomUUIDv4).slice(0, 8)}`; // This run id overrides the ambient one, so the shared run teardown never sees the volume. yield* Effect.addFinalizer(() => removeTestRunVolumes(testRunId).pipe(Effect.orDie)); const storage = yield* makeDockerDatabaseStorage({ + claims: noContainerClaims, runtime: "docker", + target: dockerTarget, stackId, instanceId: "database", instanceRoot, @@ -675,7 +819,9 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-empty-" }); const instanceRoot = path.join(root, "state", "stack", "data", "empty"); const storage = yield* makeDockerDatabaseStorage({ + claims: noContainerClaims, runtime: "docker", + target: dockerTarget, stackId: "storage-empty-test", instanceId: "empty", instanceRoot, @@ -696,6 +842,156 @@ describe("Docker database storage", { timeout: 120_000 }, () => { ).pipe(Effect.provide(NodeServices.layer)), ); + it.live( + "keeps the prior marker decodable when a real kill lands between staging and publishing an update", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-crash-" }); + const instanceRoot = path.join(root, "state", "stack", "data", "crash"); + const dataRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + const markerPath = path.join(instanceRoot, ".supabase-database-storage.json"); + const helperImage = yield* postgresImage("17"); + const storage = yield* makeDockerDatabaseStorage({ + claims: noContainerClaims, + runtime: "docker", + target: dockerTarget, + stackId: "storage-crash-test", + instanceId: "crash", + instanceRoot, + root: dataRoot, + cacheRoot, + fs, + path, + crypto, + container: yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root: instanceRoot, + }), + spawner, + }); + // Removes the real volume the seeding step below creates, rather than leaving it for + // the global test-run teardown, since a container that mounted it may not yet be + // reaped by the time that teardown runs. + yield* Effect.addFinalizer(() => storage.destroyData("17").pipe(Effect.ignore)); + yield* storage.prepare("17"); + const before = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(markerPath), + ); + expect(before.initialized).toBe(false); + if (before.backend !== "docker" || before.volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + const volume = before.volume; + // The writer below is killed, not interrupted, so its own one-off helper container (the + // PG_VERSION check's, and any the killed markInitialized retry below starts) never runs + // its own cleanup; removed here instead of leaving it attached when `destroyData` above + // (registered first, so it runs after this) tries to remove the volume. + yield* Effect.addFinalizer(() => + docker(["ps", "--all", "--quiet", "--filter", `volume=${volume}`]).pipe( + Effect.flatMap((listed) => { + const ids = listed + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); + return ids.length === 0 + ? Effect.void + : docker(["rm", "--force", ...ids]).pipe(Effect.asVoid); + }), + Effect.ignore, + ), + ); + // `markInitialized` (below, and inside the killed writer) checks this file through a + // helper container; this test is about the marker's own crash-safety, not PostgreSQL's, + // so it is seeded directly rather than through a real database launch. + yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${before.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `set -eu; mkdir -p ${quote(`/store/${before.namespace}/data`)}; printf 17 > ${quote(`/store/${before.namespace}/data/PG_VERSION`)}`, + ]); + + const writer = yield* spawner.spawn( + ChildProcess.make( + process.execPath, + [dockerStoragePublishLoopFixture, instanceRoot, dataRoot, cacheRoot], + { stdin: "ignore", stdout: "pipe", stderr: "pipe" }, + ), + ); + const ready = yield* Deferred.make(); + const stderr = yield* Ref.make(""); + const diagnostics = yield* writer.stderr.pipe( + Stream.decodeText, + Stream.runForEach((chunk) => Ref.update(stderr, (text) => text + chunk)), + Effect.forkScoped, + ); + const output = yield* writer.stdout.pipe( + Stream.decodeText, + Stream.splitLines, + Stream.tap((line) => + line === "about-to-publish" ? Deferred.succeed(ready, undefined) : Effect.void, + ), + Stream.runDrain, + Effect.forkScoped, + ); + const writerFailure = (reason: string) => + Ref.get(stderr).pipe( + Effect.flatMap((text) => + Effect.fail(new PublishBarrierError({ message: `${reason}: ${text}` })), + ), + ); + // The fixture announces this boundary itself, right before the real rename call; + // killing here lands in the staging-to-publish gap on every run, not "most of the time". + yield* Deferred.await(ready).pipe( + Effect.raceFirst( + writer.exitCode.pipe( + Effect.matchEffect({ + onFailure: (cause) => + writerFailure(`Writer exited before readiness: ${String(cause)}`), + onSuccess: (code) => writerFailure(`Writer exited before readiness (${code})`), + }), + ), + ), + Effect.timeoutOrElse({ + duration: "10 seconds", + orElse: () => writerFailure("Writer did not reach the publish boundary"), + }), + ); + yield* Effect.sync(() => process.kill(Number(writer.pid), "SIGKILL")); + yield* writer.exitCode.pipe(Effect.ignore); + yield* Fiber.join(output); + yield* Fiber.join(diagnostics); + + // The killed writer never reached the rename: the prior marker is untouched, and only + // its staging file, never a partial target, is left behind. + const afterKill = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(markerPath), + ); + expect(afterKill).toEqual(before); + expect( + (yield* fs.readDirectory(instanceRoot)).filter((entry) => entry.endsWith(".tmp")), + ).toHaveLength(1); + + yield* storage.markInitialized("17"); + const afterRetry = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(markerPath), + ); + expect(afterRetry.initialized).toBe(true); + }), + ).pipe(Effect.provide(NodeServices.layer)), + 20_000, + ); + it.live("refuses to start unmarked data and removes it through the helper on destroy", () => Effect.scoped( Effect.gen(function* () { @@ -728,10 +1024,16 @@ describe("Docker database storage", { timeout: 120_000 }, () => { "-c", "chown -R 100:101 /instance/data /instance/.supabase-snapshots; chmod -R 700 /instance/data /instance/.supabase-snapshots", ]); - const container = yield* makeContainerRuntime({ engine: "docker", root }); + const container = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root, + }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const storage = yield* makeDockerDatabaseStorage({ + claims: noContainerClaims, runtime: "docker", + target: dockerTarget, stackId: `storage-unmarked-${yield* crypto.randomUUIDv4}`, instanceId: "unmarked", instanceRoot, @@ -758,21 +1060,23 @@ describe("Docker database storage", { timeout: 120_000 }, () => { ).pipe(Effect.provide(NodeServices.layer)), ); - it.live("refuses to adopt unmarked Podman data at startup", () => + it.live("refuses to adopt unmarked Docker data at startup", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "podman-storage-unmarked-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-unmarked-" }); const storageRoot = path.join(root, "state", "stack", "data"); const instanceRoot = path.join(storageRoot, "unmarked"); const dataRoot = path.join(instanceRoot, "data"); yield* fs.makeDirectory(dataRoot, { recursive: true }); yield* fs.writeFileString(path.join(dataRoot, "PG_VERSION"), "17\n"); const storage = yield* makeDockerDatabaseStorage({ - runtime: "podman", - stackId: "storage-podman-unmarked", + claims: noContainerClaims, + runtime: "docker", + target: dockerTarget, + stackId: "storage-docker-unmarked", instanceId: "unmarked", instanceRoot, root: storageRoot, @@ -780,7 +1084,11 @@ describe("Docker database storage", { timeout: 120_000 }, () => { fs, path, crypto, - container: yield* makeContainerRuntime({ engine: "podman", root }), + container: yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root, + }), spawner: yield* ChildProcessSpawner.ChildProcessSpawner, }); @@ -806,7 +1114,11 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const instanceRoot = path.join(storageRoot, "recovery"); yield* fs.makeDirectory(instanceRoot, { recursive: true }); yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ engine: "docker", root }); + const container = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root, + }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const stackId = `storage-helper-recovery-${yield* crypto.randomUUIDv4}`; const instanceId = "recovery"; @@ -816,7 +1128,9 @@ describe("Docker database storage", { timeout: 120_000 }, () => { Effect.provideService(Scope.Scope, helperScope), ); const storage = yield* makeDockerDatabaseStorage({ + claims: noContainerClaims, runtime: "docker", + target: dockerTarget, stackId, instanceId, instanceRoot, @@ -867,7 +1181,11 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const instanceRoot = path.join(storageRoot, "database"); yield* fs.makeDirectory(instanceRoot, { recursive: true }); yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ engine: "docker", root }); + const container = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root, + }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const stackId = `storage-helper-owner-${yield* crypto.randomUUIDv4}`; const firstScope = yield* Scope.make(); @@ -895,7 +1213,9 @@ describe("Docker database storage", { timeout: 120_000 }, () => { ); const makeStorage = (helpers: DockerHelperRegistry) => makeDockerDatabaseStorage({ + claims: noContainerClaims, runtime: "docker", + target: dockerTarget, stackId, instanceId: "database", instanceRoot, @@ -959,12 +1279,18 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const instanceRoot = path.join(storageRoot, "missing"); yield* fs.makeDirectory(instanceRoot, { recursive: true }); yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ engine: "docker", root }); + const container = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root, + }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const stackId = `storage-missing-${yield* crypto.randomUUIDv4}`; const makeStorage = () => makeDockerDatabaseStorage({ + claims: noContainerClaims, runtime: "docker", + target: dockerTarget, stackId, instanceId: "missing", instanceRoot, @@ -1059,7 +1385,11 @@ describe("Docker database storage", { timeout: 120_000 }, () => { path.join(instanceRoot, ".supabase-database-ready.json"), '{"version":"17","runtime":"docker","profile":"supabase"}', ); - const container = yield* makeContainerRuntime({ engine: "docker", root }); + const container = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root, + }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; yield* docker([ "run", @@ -1072,7 +1402,9 @@ describe("Docker database storage", { timeout: 120_000 }, () => { "chown -R 100:101 /instance/data; chmod 700 /instance/data", ]); const storage = yield* makeDockerDatabaseStorage({ + claims: noContainerClaims, runtime: "docker", + target: dockerTarget, stackId: "storage-host-test", instanceId: "adopted", instanceRoot, @@ -1173,7 +1505,11 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* fs.makeDirectory(sourceRoot, { recursive: true }); yield* fs.makeDirectory(targetRoot, { recursive: true }); yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ engine: "docker", root }); + const container = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root, + }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const makeStorageWithCache = ( instanceId: string, @@ -1181,7 +1517,9 @@ describe("Docker database storage", { timeout: 120_000 }, () => { cacheRootValue: string, ) => makeDockerDatabaseStorage({ + claims: noContainerClaims, runtime: "docker", + target: dockerTarget, stackId: "storage-reopen-test", instanceId, instanceRoot, diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index f35f5d9a4f..7a693ff612 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -16,7 +16,9 @@ import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/u import { postgresVersion, resolveArtifact } from "../Artifacts.ts"; import { failureMessage } from "../internal/failure-message.ts"; import { testRunLabelArgs as readTestRunLabelArgs } from "../internal/test-run-label.ts"; -import type { ContainerRuntime } from "../runtime/Container.ts"; +import type * as Claims from "../namespace/Claims.ts"; +import * as Publication from "../namespace/Publication.ts"; +import type { ContainerRuntime, EngineTarget } from "../runtime/Container.ts"; import { composeProjectFor } from "../runtime/ContainerName.ts"; import type { DatabaseRuntime } from "../services/Database.ts"; import { @@ -120,6 +122,8 @@ const testRunLabelArgs = Effect.fn("DockerDatabaseStorage.testRunLabelArgs")(fun export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make")( (options: { readonly runtime: DatabaseRuntime; + /** The engine endpoint and identity the owner resolved once at startup. */ + readonly target: EngineTarget; readonly stackId: string; readonly instanceId: string; readonly project?: string; @@ -132,9 +136,18 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") readonly container: ContainerRuntime | undefined; readonly spawner: ChildProcessSpawnerService["Service"]; readonly helpers?: DockerHelperRegistry; + /** Journals each helper container before it is created, for the namespace's reconcile loop. */ + readonly claims: Claims.ContainerClaims; }): Effect.Effect => Effect.gen(function* () { const markerPath = options.path.join(options.instanceRoot, ".supabase-database-storage.json"); + // Atomic (stage, fsync, rename, fsync the directory), so a kill mid-write leaves the prior + // marker fully decodable rather than a partially written one. + const publishMarker = (content: string) => + Publication.publish(options.fs, options.path, { + target: markerPath, + content, + }).pipe(Effect.mapError((cause) => errorFor("marker", cause))); const composeHelperLabels = [ "--label", `com.docker.compose.project=${composeProjectFor(options.stackId, options.project)}`, @@ -202,45 +215,6 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") cacheNamespace: "native", initialized: false, }; - if (options.runtime !== "docker") { - const present = yield* options.fs - .exists(markerPath) - .pipe(Effect.mapError((cause) => errorFor("marker", cause))); - yield* options.fs - .makeDirectory(options.cacheRoot, { recursive: true }) - .pipe(Effect.mapError((cause) => errorFor("identity", cause))); - const cacheRoot = yield* options.fs - .realPath(options.cacheRoot) - .pipe(Effect.mapError((cause) => errorFor("identity", cause))); - const cacheNamespace = `cache-${(yield* hash(cacheRoot)).slice(0, 32)}`; - if (present) { - const marker = yield* options.fs.readFileString(markerPath).pipe( - Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(Marker))), - Effect.flatMap(validateMarker), - Effect.mapError((cause) => errorFor("marker", cause)), - ); - if (marker.cacheNamespace !== cacheNamespace) { - const updated = { ...marker, cacheNamespace }; - yield* options.fs - .writeFileString(markerPath, yield* encodeMarker(updated), { mode: 0o600 }) - .pipe(Effect.mapError((cause) => errorFor("marker", cause))); - return updated; - } - return marker; - } - yield* rejectUnmarkedData; - const value: Marker = { - backend: "host", - namespace: dataNamespace, - cacheNamespace, - initialized: false, - }; - const encoded = yield* encodeMarker(value); - yield* options.fs - .writeFileString(markerPath, encoded, { mode: 0o600 }) - .pipe(Effect.mapError((cause) => errorFor("marker", cause))); - return value; - } let resolved = yield* resolveDaemon(false); const markerPresent = yield* options.fs .exists(markerPath) @@ -305,10 +279,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ...validMarker, cacheNamespace: `cache-${resolved.cacheDigest.slice(0, 32)}`, }; - if (updated !== validMarker) - yield* options.fs - .writeFileString(markerPath, yield* encodeMarker(updated), { mode: 0o600 }) - .pipe(Effect.mapError((cause) => errorFor("marker", cause))); + if (updated !== validMarker) yield* publishMarker(yield* encodeMarker(updated)); return updated; } yield* rejectUnmarkedData; @@ -351,9 +322,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") Effect.asVoid, ); } - yield* options.fs - .writeFileString(markerPath, yield* encodeMarker(value), { mode: 0o600 }) - .pipe(Effect.mapError((cause) => errorFor("marker", cause))); + yield* publishMarker(yield* encodeMarker(value)); return value; }); yield* Ref.set(selectedCache, value); @@ -370,7 +339,11 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") "process.arg_count": args.length, }); const child = yield* options.spawner - .spawn(ChildProcess.make(options.runtime, args, { stdin: "ignore" })) + .spawn( + ChildProcess.make(options.runtime, [...options.target.argv, ...args], { + stdin: "ignore", + }), + ) .pipe(Effect.mapError((cause) => errorFor("engine", cause))); const [stdout, stderr, code] = yield* Effect.all( [ @@ -414,9 +387,11 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") : Effect.fail(cause), ), ); - const probeDaemonIdentity: Effect.Effect = + // The daemon id is the owner's own pinned target, resolved once at startup; only the + // version, which that target's identity says nothing about, still needs the daemon itself. + const probeDaemonVersion: Effect.Effect = Effect.gen(function* () { - const daemonId = (yield* engineCommand(["info", "--format", "{{.ID}}"])).trim(); + const daemonId = options.target.daemonId; const version = yield* engineCommand([ "version", "--format", @@ -425,7 +400,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") const [clientVersion, serverVersion] = version.split("|"); const clientMajor = parseMajor(clientVersion ?? ""); const serverMajor = parseMajor(serverVersion ?? ""); - if (daemonId.length === 0 || clientMajor === undefined || serverMajor === undefined) + if (clientMajor === undefined || serverMajor === undefined) return yield* errorFor("engine", "Docker returned an invalid version"); const identity = { daemonId, clientMajor, serverMajor }; // The next process, including a shadow diff, reuses this instead of asking the daemon. @@ -486,7 +461,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") } } } - const resolved = yield* describe(yield* probeDaemonIdentity, true, false); + const resolved = yield* describe(yield* probeDaemonVersion, true, false); yield* Ref.set(daemonIdentityRef, resolved); return resolved; }); @@ -546,6 +521,11 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") /no such container/iu.test(cause.message) ? Effect.void : Effect.fail(cause), ), ); + // Cleanup authority now belongs entirely to this normal teardown; a crash before + // this point still leaves the claim for the next owner's reconcile loop. + yield* options.claims + .unclaim(current) + .pipe(Effect.mapError((cause) => errorFor("claim", cause))); yield* Ref.set(helperId, undefined); yield* Ref.set(helperImage, undefined); yield* Ref.set(helperCleanupPending, false); @@ -603,7 +583,15 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") const name = `supabase-db-helper-${token}`; const testRunLabel = yield* testRunLabelArgs(); // Register the deterministic owned name before the remote create starts so an - // interrupted docker run can still be removed by the same scope. + // interrupted docker run can still be removed by the same scope, and journal it so a + // SIGKILL that skips that scope still leaves it recoverable by the next reconcile. + // Required, the same as launch: a container must never be created with an + // unverifiable daemon identity, since a crashed owner could then leave a claim no + // reconcile can ever trust. + const resolvedDaemonId = options.target.daemonId; + yield* options.claims + .claim(name, resolvedDaemonId) + .pipe(Effect.mapError((cause) => errorFor("claim", cause))); yield* Ref.set(helperId, name); yield* Ref.set(helperImage, image); const created = yield* engineCommand([ @@ -655,6 +643,9 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") Effect.catchTag("DockerDatabaseStorageError", (cause) => missingContainer(cause.message) ? Effect.void : Effect.fail(cause), ), + Effect.andThen( + options.claims.unclaim(id).pipe(Effect.mapError((cause) => errorFor("claim", cause))), + ), Effect.asVoid, ); // A container left in `created` cannot exec; remove it and start another. @@ -697,6 +688,14 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") .prepareImage(image) .pipe(Effect.mapError((cause) => errorFor("helper", cause))); const testRunLabel = yield* testRunLabelArgs(); + // Journaled before the create starts, so a SIGKILL leaves this shared helper's identity + // recoverable by the next reconcile instead of leaked. Required, the same as launch: a + // container must never be created with an unverifiable daemon identity, since a crashed + // owner could then leave a claim no reconcile can ever trust. + const resolvedDaemonId = options.target.daemonId; + yield* options.claims + .claim(name, resolvedDaemonId) + .pipe(Effect.mapError((cause) => errorFor("claim", cause))); return yield* Effect.uninterruptible( engineCommand([ "run", @@ -829,11 +828,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* removeReadyMarker; }); const writeMarker = (marker: Marker) => - encodeMarker(marker).pipe( - Effect.flatMap((encoded) => - options.fs.writeFileString(markerPath, encoded, { mode: 0o600 }), - ), - ); + encodeMarker(marker).pipe(Effect.flatMap((encoded) => publishMarker(encoded))); const setup = Effect.fn("DockerDatabaseStorage.prepare")((version: string) => Effect.gen(function* () { yield* selected; @@ -1096,43 +1091,3 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") }; }), ); - -/** Shell commands that delete the Docker-volume data recorded by the database instances under a stack's data root. */ -export const volumeDataCleanupCommands = Effect.fn( - "DockerDatabaseStorage.volumeDataCleanupCommands", -)( - function* (options: { - readonly engine: "docker" | "podman"; - readonly root: string; - readonly fs: FileSystem.FileSystem; - readonly path: Path.Path; - }) { - if (!(yield* options.fs.exists(options.root))) return []; - const markers: Array = []; - for (const entry of yield* options.fs.readDirectory(options.root)) { - const markerPath = options.path.join(options.root, entry, ".supabase-database-storage.json"); - if (!(yield* options.fs.exists(markerPath))) continue; - const marker = yield* options.fs - .readFileString(markerPath) - .pipe(Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(Marker)))); - if (marker.backend !== "docker") continue; - if ( - marker.volume === undefined || - !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/u.test(marker.volume) || - !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/u.test(marker.namespace) - ) - return yield* errorFor( - "destroy", - `Recorded database storage identity is invalid: ${markerPath}`, - ); - markers.push(marker); - } - if (markers.length === 0) return []; - const { image } = yield* resolveArtifact({ service: "database" }); - return markers.map( - (marker) => - `${options.engine} run --rm --mount ${shellQuote(`type=volume,src=${marker.volume},dst=/store`)} ${shellQuote(image)} /bin/sh -c ${shellQuote(`rm -rf /store/${marker.namespace}`)}`, - ); - }, - Effect.mapError((cause) => errorFor("destroy", cause)), -); diff --git a/packages/stack/src/testing.integration.test.ts b/packages/stack/src/testing.integration.test.ts index 8f92f1b45e..a193eb6a99 100644 --- a/packages/stack/src/testing.integration.test.ts +++ b/packages/stack/src/testing.integration.test.ts @@ -102,19 +102,21 @@ it.live( yield* test.checkpoint("0"); const root = path.join(stateRoot, test.stack.id); - const owners = (yield* fs.readDirectory(root, { recursive: true })).filter((file) => - file.endsWith(".supabase-database-owner.json"), + const readyMarkers = (yield* fs.readDirectory(root, { recursive: true })).filter((file) => + file.endsWith(".supabase-database-ready.json"), ); - expect(owners).toHaveLength(1); - const ownerFile = path.join(root, owners[0]!); - const originalMarker = yield* fs.readFileString(ownerFile); - // Released before the stack's teardown, which destroys only data carrying its own marker. + expect(readyMarkers).toHaveLength(1); + const instanceRoot = path.dirname(path.join(root, readyMarkers[0]!)); + const outside = yield* fs.makeTempDirectoryScoped({ prefix: "reset-failure-outside-" }); + // Ownership is by location: replacing the owned root with a symlink makes resetData's + // destroy step refuse, simulating a failure partway through reset before the checkpoint + // restore runs. Released before the stack's teardown, which tolerates a root that is simply + // absent. yield* Effect.acquireRelease( - fs.writeFileString( - ownerFile, - `{"stackId":"not-this-stack","instanceId":"not-this-instance"}`, - ), - () => fs.writeFileString(ownerFile, originalMarker).pipe(Effect.orDie), + fs + .remove(instanceRoot, { recursive: true, force: true }) + .pipe(Effect.andThen(fs.symlink(outside, instanceRoot))), + () => fs.remove(instanceRoot, { force: true }).pipe(Effect.orDie), ); const failure = yield* test.reset("0").pipe(Effect.flip); diff --git a/packages/stack/src/testing.ts b/packages/stack/src/testing.ts index b92c16d860..2be287bc91 100644 --- a/packages/stack/src/testing.ts +++ b/packages/stack/src/testing.ts @@ -114,7 +114,7 @@ export interface TestStack< } const runtimeOverride = Config.option( - Config.literals(["native", "docker", "podman"], "SUPABASE_STACK_TEST_RUNTIME"), + Config.literals(["native", "docker"], "SUPABASE_STACK_TEST_RUNTIME"), ); const testFailure = (operation: string) => (cause: unknown) => stackError(operation, cause); diff --git a/packages/stack/src/whole-stack.docker.lifecycle.e2e.test.ts b/packages/stack/src/whole-stack.docker.lifecycle.e2e.test.ts index 53aa5d5dd0..47a825ef47 100644 --- a/packages/stack/src/whole-stack.docker.lifecycle.e2e.test.ts +++ b/packages/stack/src/whole-stack.docker.lifecycle.e2e.test.ts @@ -1,5 +1,5 @@ import { it } from "@effect/vitest"; -import { allEager, defaultLifecycle } from "../tests/whole-stack/scenarios.ts"; +import { allEager, defaultLifecycle, writeConfinement } from "../tests/whole-stack/scenarios.ts"; import { run } from "../tests/whole-stack/fixture.ts"; it.live( @@ -10,3 +10,8 @@ it.live( }, ); it.live("Docker: all services eager", () => run(allEager("docker")), { timeout: 15 * 60_000 }); +it.live( + "Docker: writes stay confined to the state root and artifact cache", + () => run(writeConfinement("docker")), + { timeout: 15 * 60_000 }, +); diff --git a/packages/stack/src/whole-stack.native.lifecycle.e2e.test.ts b/packages/stack/src/whole-stack.native.lifecycle.e2e.test.ts index d78a02f169..e71396083e 100644 --- a/packages/stack/src/whole-stack.native.lifecycle.e2e.test.ts +++ b/packages/stack/src/whole-stack.native.lifecycle.e2e.test.ts @@ -1,5 +1,5 @@ import { it } from "@effect/vitest"; -import { allEager, defaultLifecycle } from "../tests/whole-stack/scenarios.ts"; +import { allEager, defaultLifecycle, writeConfinement } from "../tests/whole-stack/scenarios.ts"; import { run } from "../tests/whole-stack/fixture.ts"; it.live( @@ -10,3 +10,8 @@ it.live( }, ); it.live("native: all services eager", () => run(allEager("native")), { timeout: 15 * 60_000 }); +it.live( + "native: writes stay confined to the state root and artifact cache", + () => run(writeConfinement("native")), + { timeout: 15 * 60_000 }, +); diff --git a/packages/stack/tests/claims.ts b/packages/stack/tests/claims.ts new file mode 100644 index 0000000000..5b9bbb0a9a --- /dev/null +++ b/packages/stack/tests/claims.ts @@ -0,0 +1,15 @@ +import { Effect } from "effect"; +import type { ContainerClaims, DirectoryClaims } from "../src/namespace/Claims.ts"; + +/** + * No-op claims for tests that exercise container or directory creation without exercising the + * namespace's reconcile loop itself (covered separately by the namespace's own test suite). + */ +export const noContainerClaims: ContainerClaims = { + claim: () => Effect.void, + unclaim: () => Effect.void, +}; +export const noDirectoryClaims: DirectoryClaims = { + claim: () => Effect.void, + unclaim: () => Effect.void, +}; diff --git a/packages/stack/tests/compiled-dispatch-fixture.ts b/packages/stack/tests/compiled-dispatch-fixture.ts index ff2f35a3fd..40d950ab50 100644 --- a/packages/stack/tests/compiled-dispatch-fixture.ts +++ b/packages/stack/tests/compiled-dispatch-fixture.ts @@ -65,6 +65,19 @@ if (!(await runHostProcessIfDispatched(argv)) && !(await runNativeProcessIfDispa "sh", pidMarker, ], + environment: { + values: { + HOME: "/tmp", + TMPDIR: "/tmp", + TMP: "/tmp", + TEMP: "/tmp", + XDG_CACHE_HOME: "/tmp", + XDG_CONFIG_HOME: "/tmp", + XDG_DATA_HOME: "/tmp", + XDG_STATE_HOME: "/tmp", + DENO_DIR: "/tmp", + }, + }, gracefulStopSignal: "SIGTERM", gracefulStopTimeout: "2 seconds", }); diff --git a/packages/stack/tests/docker-fixture.integration.test.ts b/packages/stack/tests/docker-fixture.integration.test.ts index 17303cf6ab..8c8e7c4943 100644 --- a/packages/stack/tests/docker-fixture.integration.test.ts +++ b/packages/stack/tests/docker-fixture.integration.test.ts @@ -6,6 +6,8 @@ import { Context, Effect, FileSystem, Layer, Redacted, Schema, Stream } from "ef import { makeService } from "../src/Service.ts"; import { makeDatabase, type BackendEndpoint } from "../src/services/Database.ts"; import { makeDockerDatabaseRoot } from "./docker-fixture.ts"; +import { noContainerClaims, noDirectoryClaims } from "./claims.ts"; +import { dockerEngineTarget } from "./engine-target.ts"; const marker = Schema.Struct({ backend: Schema.Literal("docker"), volume: Schema.String }); const password = Redacted.make("supabase-test-password"); @@ -73,6 +75,9 @@ describe("Docker database fixture isolation", { timeout: 180_000 }, () => { root, cacheRoot: `${root}/cache`, runtime: "docker", + engineTarget: dockerEngineTarget, + containerClaims: noContainerClaims, + directoryClaims: noDirectoryClaims, }); const [first, second] = yield* Effect.all([make(firstRoot), make(secondRoot)]); const [firstService, secondService] = yield* Effect.all([ diff --git a/packages/stack/tests/docker-storage-publish-loop-fixture.ts b/packages/stack/tests/docker-storage-publish-loop-fixture.ts new file mode 100644 index 0000000000..f2a0a36497 --- /dev/null +++ b/packages/stack/tests/docker-storage-publish-loop-fixture.ts @@ -0,0 +1,64 @@ +import { NodeServices } from "@effect/platform-node"; +import { Console, Crypto, Effect, FileSystem, Layer, Path } from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import { makeContainerRuntime } from "../src/runtime/Container.ts"; +import { makeDockerDatabaseStorage } from "../src/storage/DockerDatabaseStorage.ts"; +import { noContainerClaims } from "./claims.ts"; + +const [instanceRoot, root, cacheRoot] = process.argv.slice(2); +if (instanceRoot === undefined || root === undefined || cacheRoot === undefined) + throw new Error("instanceRoot, root or cacheRoot missing"); + +/** + * Announces readiness on stdout the instant it is about to replace the storage marker (the real + * staging-to-publish boundary Publication.publish uses), then hangs so a caller can kill this + * process exactly there on every run. + */ +const barrierFs = Layer.effect( + FileSystem.FileSystem, + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + return { + ...fs, + rename: (from: string, to: string) => + to.endsWith(".supabase-database-storage.json") + ? Console.log("about-to-publish").pipe(Effect.andThen(Effect.never)) + : fs.rename(from, to), + }; + }), +); + +const program = Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + // Matches the caller's own target exactly, so this fixture's identity agrees with the + // marker the caller's `prepare` already published through it. + const target = { engine: "docker" as const, argv: [], daemonId: "test-daemon-id" }; + const container = yield* makeContainerRuntime({ + claims: noContainerClaims, + target, + root: instanceRoot, + }); + const storage = yield* makeDockerDatabaseStorage({ + claims: noContainerClaims, + runtime: "docker", + target, + stackId: "storage-crash-test", + instanceId: "crash", + instanceRoot, + root, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + yield* storage.markInitialized("17"); + }), +).pipe(Effect.provide(barrierFs.pipe(Layer.provideMerge(NodeServices.layer)))); + +await Effect.runPromise(program); diff --git a/packages/stack/tests/engine-target.ts b/packages/stack/tests/engine-target.ts new file mode 100644 index 0000000000..3e5fb0eb84 --- /dev/null +++ b/packages/stack/tests/engine-target.ts @@ -0,0 +1,13 @@ +import { NodeServices } from "@effect/platform-node"; +import { Effect } from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import { resolveEngineTarget, type EngineTarget } from "../src/runtime/Container.ts"; + +/** + * Resolves the real docker engine target once per test file, for `it.live` tests that exercise a + * service catalog or a database against the real daemon rather than a fake one. + */ +export const dockerEngineTarget: EngineTarget = await Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* resolveEngineTarget(spawner); +}).pipe(Effect.provide(NodeServices.layer), Effect.runPromise); diff --git a/packages/stack/tests/owner-rpc.ts b/packages/stack/tests/owner-rpc.ts index 50652a6db0..933d4571b3 100644 --- a/packages/stack/tests/owner-rpc.ts +++ b/packages/stack/tests/owner-rpc.ts @@ -2,6 +2,7 @@ import { Context, Effect, Layer } from "effect"; import { RpcTest } from "effect/unstable/rpc"; import * as Owner from "../src/Owner.ts"; import { OwnerRpc } from "../src/Rpc.ts"; +import type { EngineTarget } from "../src/runtime/Container.ts"; import { Service as StateService } from "../src/StackNamespace.ts"; import type { Interface as StateInterface, SavedStack } from "../src/StackNamespace.ts"; @@ -11,6 +12,7 @@ export const ownerFor = (options: { readonly state: StateInterface; readonly root: string; readonly cacheRoot: string; + readonly engineTarget?: EngineTarget; }) => Effect.gen(function* () { const { state, ...layerOptions } = options; diff --git a/packages/stack/tests/whole-stack/confinement.ts b/packages/stack/tests/whole-stack/confinement.ts new file mode 100644 index 0000000000..dc882349e1 --- /dev/null +++ b/packages/stack/tests/whole-stack/confinement.ts @@ -0,0 +1,162 @@ +import { expect } from "@effect/vitest"; +import { Effect, FileSystem, Option, Path } from "effect"; + +interface TreeEntry { + readonly isFile: boolean; + readonly size: bigint; + readonly mtimeMs: number; +} + +export type Tree = ReadonlyMap; + +/** Snapshots every path under `root`, or an empty tree when `root` does not exist yet. */ +export const snapshotTree = Effect.fn("Confinement.snapshotTree")(function* (root: string) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + if (!(yield* fs.exists(root))) return new Map(); + const entries = yield* fs.readDirectory(root, { recursive: true }); + const stats = yield* Effect.forEach( + entries, + (entry) => { + const full = path.join(root, entry); + return fs.stat(full).pipe( + Effect.map((info): readonly [string, TreeEntry] => [ + full, + { + isFile: info.type === "File", + size: info.size, + mtimeMs: Option.match(info.mtime, { + onNone: () => 0, + onSome: (date) => date.getTime(), + }), + }, + ]), + // A path can disappear between listing and stat (e.g. a service's own temp file); treat it + // as absent rather than failing the whole snapshot. + Effect.orElseSucceed(() => [full, { isFile: false, size: 0n, mtimeMs: 0 }] as const), + ); + }, + { concurrency: 8 }, + ); + return new Map(stats); +}); + +export interface ConfinementViolation { + readonly path: string; + readonly kind: "created" | "modified"; +} + +/** Flags every path that is new or changed between two snapshots of the same root. */ +export const diffTrees = (before: Tree, after: Tree): ReadonlyArray => { + const violations: Array = []; + for (const [entryPath, entry] of after) { + const previous = before.get(entryPath); + if (previous === undefined) violations.push({ path: entryPath, kind: "created" }); + else if (entry.isFile && (entry.size !== previous.size || entry.mtimeMs !== previous.mtimeMs)) + violations.push({ path: entryPath, kind: "modified" }); + } + return violations; +}; + +/** Fails, listing every offending path, when a violation falls outside the allowed roots. */ +export const assertConfinedTo = ( + violations: ReadonlyArray, + allowedRoots: ReadonlyArray, +): void => { + const isAllowed = (candidate: string) => + allowedRoots.some((root) => candidate === root || candidate.startsWith(`${root}/`)); + const offending = violations.filter((violation) => !isAllowed(violation.path)); + expect( + offending, + `Stack wrote outside its allowed roots (${allowedRoots.join(", ")}):\n${offending + .map((violation) => ` ${violation.kind}: ${violation.path}`) + .join("\n")}`, + ).toEqual([]); +}; + +/** + * Negative control: proves `assertConfinedTo` actually flags an unauthorized write under `root`, + * for both a write that persists and one created and then removed again before the next snapshot + * (the gap a before/after-only check would otherwise miss). Cleans up after itself so it leaves no + * trace for the scenario's own, later checks. + */ +export const proveConfinementDetectsViolations = Effect.fn( + "Confinement.proveConfinementDetectsViolations", +)(function* (root: string, baseline: Tree, allowedRoots: ReadonlyArray) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const persistentMarker = path.join(root, ".confinement-control-persistent"); + const transientMarker = path.join(root, ".confinement-control-transient"); + yield* fs.writeFileString(persistentMarker, "control"); + yield* fs.writeFileString(transientMarker, "control"); + const withBothMarkers = diffTrees(baseline, yield* snapshotTree(root)); + expect(withBothMarkers.map((violation) => violation.path).sort()).toEqual( + [persistentMarker, transientMarker].sort(), + ); + expect(() => assertConfinedTo(withBothMarkers, allowedRoots)).toThrow(); + yield* fs.remove(transientMarker, { force: true }); + const afterTransientRemoved = diffTrees(baseline, yield* snapshotTree(root)); + // The residual gap: a write made and undone strictly between two snapshot points is invisible + // to a diff taken only afterward, unlike the persistent one. + expect(afterTransientRemoved.map((violation) => violation.path)).toEqual([persistentMarker]); + yield* fs.remove(persistentMarker, { force: true }); +}); + +/** + * Environment names the stack, its workloads, or the test harness's own Bun runtime resolve a + * user-level root from. + */ +const sandboxEnvNames = [ + "HOME", + "TMPDIR", + "TMP", + "TEMP", + "XDG_CACHE_HOME", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + "XDG_STATE_HOME", + // Bun's own JIT transpile cache; a test-harness concern, not a stack confinement one. + "BUN_RUNTIME_TRANSPILER_CACHE_PATH", + // Pre-resolved, in the real environment, to the endpoint the engine's own context store would + // have answered: once set, the owner's pinned target resolution skips that store entirely, so + // it never needs `~/.docker` under the sandboxed HOME below. Exposing that real config directory + // instead would also expose its credential store, which a real pull would then invoke under the + // sandboxed HOME, writing outside what this test allows. + "DOCKER_HOST", +] as const; + +type SandboxEnvName = (typeof sandboxEnvNames)[number]; + +export type SandboxEnvironment = Readonly>; + +/** + * Runs `use` with the given variables set on the process environment, so a detached stack owner + * spawned during `use` inherits them, and restores the previous values once `use` completes. + */ +export const withSandboxEnvironment = ( + values: SandboxEnvironment, + use: Effect.Effect, +): Effect.Effect => + Effect.acquireUseRelease( + Effect.sync(() => { + const previous: Partial> = {}; + for (const name of sandboxEnvNames) { + // oxlint-disable-next-line effecttsgo/process-env-in-effect, effecttsgo/process-env -- the detached stack owner and the harness's own Bun process inherit these at spawn time; this is not application config. + previous[name] = process.env[name]; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. + process.env[name] = values[name]; + } + return previous; + }), + () => use, + (previous) => + Effect.sync(() => { + for (const name of sandboxEnvNames) { + const value = previous[name]; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. + if (value === undefined) delete process.env[name]; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. + else process.env[name] = value; + } + }), + ); diff --git a/packages/stack/tests/whole-stack/scenarios.ts b/packages/stack/tests/whole-stack/scenarios.ts index 5edcc136fa..7a1cffe511 100644 --- a/packages/stack/tests/whole-stack/scenarios.ts +++ b/packages/stack/tests/whole-stack/scenarios.ts @@ -1,6 +1,7 @@ import { expect } from "@effect/vitest"; -import { Cause, Effect, FileSystem, Ref, Schema } from "effect"; +import { Cause, Effect, FileSystem, Path, Ref, Schema } from "effect"; import { open } from "../../src/effect.ts"; +import { runDocker } from "../docker-fixture.ts"; import { allMembers, clearStackOwner, @@ -28,6 +29,13 @@ import { subscribeRealtime } from "./websocket.ts"; import { watchRecoveryMail } from "./realtime-mail.ts"; import { assertWorkloadsGone, captureWorkloads } from "./workloads.ts"; import { assertOwnerExited, captureOwnerPid } from "../owner.ts"; +import { + assertConfinedTo, + diffTrees, + proveConfinementDetectsViolations, + snapshotTree, + withSandboxEnvironment, +} from "./confinement.ts"; const withFixture = ( runtime: Runtime, @@ -526,3 +534,89 @@ export const parallel = (runtime: Runtime) => yield* exerciseStack(right, "parallel-right-after"); }), ); + +/** + * The real environment's own docker endpoint, read here (not through the owner's own pinned-target + * resolution) purely so {@link writeConfinement} can hand the sandboxed owner a `DOCKER_HOST` that + * works without its own `~/.docker` client state. + */ +const resolveRealDockerHost = Effect.gen(function* () { + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- reads the real environment before it is sandboxed below. + const configured = process.env.DOCKER_HOST; + if (configured !== undefined && configured.length > 0) return configured; + const name = (yield* runDocker(["context", "show"])).output.trim(); + const host = (yield* runDocker([ + "context", + "inspect", + name, + "--format", + "{{.Endpoints.docker.Host}}", + ])).output.trim(); + if (host.length === 0) return yield* Effect.die(`Context ${name} has no docker endpoint`); + return host; +}); + +/** + * Proves a running stack writes only inside its state root and artifact cache (plus the project + * directory it was handed): points HOME, TMPDIR/TMP/TEMP and the XDG roots at fresh, empty + * directories for the stack and its detached owner, then diffs those directories before and after + * a full start/exercise/stop/destroy cycle. + */ +export const writeConfinement = (runtime: Runtime) => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const arena = yield* fs.makeTempDirectoryScoped({ prefix: "stack-confinement-arena-" }); + const home = path.join(arena, "home"); + const tmp = path.join(arena, "tmp"); + // Bun's own transpile cache is a test-harness concern, not a stack confinement one: it gets + // its own allowed root under `tmp` rather than being allow-listed inside `home`. + const bunTranspilerCache = path.join(tmp, "bun-transpiler-cache"); + yield* fs.makeDirectory(home, { recursive: true }); + yield* fs.makeDirectory(bunTranspilerCache, { recursive: true }); + // Resolved against the real environment before HOME below points at the empty sandbox: the + // owner's own `DOCKER_HOST`-pinned resolution then skips the engine's context store + // entirely, so it never needs `~/.docker` under the sandbox. The docker CLI's own client + // state (config, credential helpers) is outside this test's confinement contract; narrowing + // the sandboxed owner to a `DOCKER_HOST` pin instead of its real context is an accepted + // tradeoff for that, not something this test tries to confine. + const dockerHost = runtime === "docker" ? yield* resolveRealDockerHost : undefined; + yield* withSandboxEnvironment( + { + HOME: home, + TMPDIR: tmp, + TMP: tmp, + TEMP: tmp, + XDG_CACHE_HOME: path.join(home, ".cache"), + XDG_CONFIG_HOME: path.join(home, ".config"), + XDG_DATA_HOME: path.join(home, ".local", "share"), + XDG_STATE_HOME: path.join(home, ".local", "state"), + BUN_RUNTIME_TRANSPILER_CACHE_PATH: bunTranspilerCache, + DOCKER_HOST: dockerHost ?? "", + }, + Effect.gen(function* () { + const beforeHome = yield* snapshotTree(home); + const beforeTmp = yield* snapshotTree(tmp); + const fixture = yield* wholeStack(runtime); + yield* clearIdleTimers(fixture); + yield* fixture.stack.composition.start; + yield* exerciseStack(fixture, "confinement"); + // Allowed roots the named entries below cover; add one line here to extend them. + const allowedRoots = [fixture.root, fixture.locations.cacheRoot, bunTranspilerCache]; + // While the stack is still running, after it has been exercised: a before/after-only + // check misses anything a service writes and removes again before shutdown. + const duringHome = yield* snapshotTree(home); + const duringTmp = yield* snapshotTree(tmp); + assertConfinedTo(diffTrees(beforeHome, duringHome), allowedRoots); + assertConfinedTo(diffTrees(beforeTmp, duringTmp), allowedRoots); + yield* proveConfinementDetectsViolations(home, duringHome, allowedRoots); + yield* stopWithDiagnostics(fixture); + yield* fixture.stack.destroy; + yield* clearStackOwner(fixture); + assertConfinedTo(diffTrees(beforeHome, yield* snapshotTree(home)), allowedRoots); + assertConfinedTo(diffTrees(beforeTmp, yield* snapshotTree(tmp)), allowedRoots); + }), + ); + }), + ); From bf602c3786228e991c2a8bab99d634ea399a7931 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sat, 3 Oct 2026 19:49:29 +0200 Subject: [PATCH 014/142] fix(stack): keep caller-owned mounts owned by the host user and gate the Docker reconcile test --- .../stack/src/namespace/Namespace.e2e.test.ts | 3 +- packages/stack/src/runtime/Container.ts | 3 ++ packages/stack/src/services/ProcessRecipe.ts | 31 +++++++++++++++---- .../StorageGateway.integration.test.ts | 22 +++++++++++-- 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/packages/stack/src/namespace/Namespace.e2e.test.ts b/packages/stack/src/namespace/Namespace.e2e.test.ts index 04aceb258b..d25b24b405 100644 --- a/packages/stack/src/namespace/Namespace.e2e.test.ts +++ b/packages/stack/src/namespace/Namespace.e2e.test.ts @@ -340,7 +340,8 @@ const docker = Effect.fn("Namespace.e2e.docker")((args: ReadonlyArray) = ), ); -it.live.skipIf(process.platform === "win32")( +// Needs a real Docker daemon, which only the Linux CI runner has. +it.live.skipIf(process.platform === "win32" || process.platform === "darwin")( "removes exactly the containers a SIGKILLed owner's claims recorded, with no manual cleanup", () => Effect.scoped( diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index 3979097fb4..b64649e782 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -57,6 +57,8 @@ interface ContainerSpec { readonly stopGraceSeconds?: number; /** Signal `docker stop` sends first; omitted uses the image's stop signal. */ readonly stopSignal?: "SIGINT" | "SIGTERM"; + /** `uid:gid` the container runs as, overriding the image's own user. */ + readonly user?: string; } export interface ContainerProcess { @@ -689,6 +691,7 @@ export const makeContainerRuntime = (options: { "--label", `com.docker.compose.service=${composeService}`, ...(oneOff ? ["--label", "com.docker.compose.oneoff=True"] : []), + ...(spec.user === undefined ? [] : ["--user", spec.user]), "--env-file", envPath, ...(spec.mounts ?? []).flatMap((mount) => [ diff --git a/packages/stack/src/services/ProcessRecipe.ts b/packages/stack/src/services/ProcessRecipe.ts index ea4433b4e3..0b54af3536 100644 --- a/packages/stack/src/services/ProcessRecipe.ts +++ b/packages/stack/src/services/ProcessRecipe.ts @@ -59,6 +59,22 @@ interface RecipeMount { readonly readOnly: boolean; } +/** `uid:gid` a container runs as when it writes into a borrowed host path, so the caller keeps ownership. */ +const hostUser: string | undefined = + process.getuid === undefined || process.getgid === undefined + ? undefined + : `${process.getuid()}:${process.getgid()}`; + +/** The host-user override for a launch whose mounts write into one of its recipe's caller paths. */ +const userForMounts = ( + mounts: ReadonlyArray, + callerPaths: ReadonlyArray, +): string | undefined => + hostUser !== undefined && + mounts.some((mount) => !mount.readOnly && callerPaths.includes(mount.source)) + ? hostUser + : undefined; + export interface StartupCommand { readonly args: ReadonlyArray; readonly nativeExecutable?: string; @@ -780,6 +796,7 @@ export const makeProcessRecipe = entrypoint: command.entrypoint, args: command.args, mounts: command.mounts, + user: userForMounts(command.mounts, spec.callerPaths?.(context.config) ?? []), }) .pipe( Effect.catchTag("ContainerLaunchError", ({ failure, process }) => @@ -819,6 +836,12 @@ export const makeProcessRecipe = runtime: runtimeFromContainer(startupProcess), }); } + const launchMounts = [ + ...(yield* spec.mounts(context.config, { container: true })), + ...(spec.instanceDirectory === true + ? [{ source: instanceRoot, target: containerInstancePath, readOnly: false }] + : []), + ]; const launched = yield* deps.container .launch({ image: resolved.image, @@ -829,12 +852,8 @@ export const makeProcessRecipe = env: yield* spec.env(context.config, containerDesired, true, containerInstanceDir), entrypoint: spec.containerEntrypoint?.(context.config), args: yield* spec.args(context.config, containerDesired, { container: true }), - mounts: [ - ...(yield* spec.mounts(context.config, { container: true })), - ...(spec.instanceDirectory === true - ? [{ source: instanceRoot, target: containerInstancePath, readOnly: false }] - : []), - ], + mounts: launchMounts, + user: userForMounts(launchMounts, spec.callerPaths?.(context.config) ?? []), ports: [...containerDesired.values()].map((endpoint) => endpoint.port), }) .pipe( diff --git a/packages/stack/src/services/StorageGateway.integration.test.ts b/packages/stack/src/services/StorageGateway.integration.test.ts index daa43a4e8c..2813cbbb22 100644 --- a/packages/stack/src/services/StorageGateway.integration.test.ts +++ b/packages/stack/src/services/StorageGateway.integration.test.ts @@ -1,6 +1,6 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Context, Crypto, Effect, FileSystem, Layer, Path, Redacted } from "effect"; +import { Context, Crypto, Effect, FileSystem, Layer, Option, Path, Redacted } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { tmpdir } from "node:os"; import { @@ -98,7 +98,7 @@ const serveStorage = Effect.fnUntraced(function* (runtime: SavedStack["runtime"] const { url } = yield* owner.rpc.credentials({ id: storage.id, from: "host" }); if (url === undefined) return yield* Effect.die("Storage gateway URL missing"); const { serviceRoleKey } = yield* owner.getStackCredentials; - return { url, serviceRoleKey }; + return { url, serviceRoleKey, storageRoot }; }); for (const runtime of ["native", "docker"] as const) @@ -108,7 +108,7 @@ for (const runtime of ["native", "docker"] as const) Effect.scoped( Effect.gen(function* () { const client = yield* HttpClient.HttpClient; - const { url, serviceRoleKey } = yield* serveStorage(runtime); + const { url, serviceRoleKey, storageRoot } = yield* serveStorage(runtime); const bucket = "gateway"; const createBucket = yield* client.execute( @@ -162,6 +162,22 @@ for (const runtime of ["native", "docker"] as const) ); expect(patch.status, yield* patch.text).toBe(204); expect(patch.headers["upload-offset"]).toBe("5"); + + if (runtime === "docker") { + // The container writes into storageRoot, a borrowed host directory; the host user + // must still own what it created there, not the engine's own root (see Container's + // `user` field). + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const entries = yield* fs.readDirectory(storageRoot, { recursive: true }); + const stats = yield* Effect.forEach(entries, (entry) => + fs.stat(path.join(storageRoot, entry)), + ); + const files = stats.filter((info) => info.type === "File"); + expect(files.length).toBeGreaterThan(0); + for (const info of files) + expect(Option.getOrUndefined(info.uid)).toBe(process.getuid?.()); + } }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 180_000 }, From ba2ad4585295440a0b716b9716562c89cb5edc6c Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sun, 4 Oct 2026 01:36:43 +0200 Subject: [PATCH 015/142] refactor(stack): publish native artifacts as immutable pinned generations --- .../bundled-postgres-client.ts | 6 +- .../experimental/stack/start/SIDE_EFFECTS.md | 12 +- packages/stack/README.md | 4 +- packages/stack/src/Artifacts.ts | 107 +- ...dRunner.initialization.integration.test.ts | 8 +- packages/stack/src/host/CommandRunner.ts | 8 +- packages/stack/src/internal/artifacts.ts | 1 + packages/stack/src/namespace/Lease.ts | 3 +- packages/stack/src/namespace/Pin.ts | 80 ++ .../stack/src/namespace/drivers/Sqlite.ts | 49 +- .../stack/src/namespace/drivers/sqlite-pin.ts | 39 + .../stack/src/preparation/ArtifactStore.ts | 913 +++++++------ .../preparation/artifacts.integration.test.ts | 1170 +++++++++-------- .../slim-services.integration.test.ts | 62 +- packages/stack/src/runtime/NativeProcess.ts | 7 + packages/stack/src/runtime/native-launcher.ts | 128 +- packages/stack/src/runtime/postgres-user.ts | 35 +- packages/stack/src/services/Database.ts | 36 +- .../ProcessRecipe.integration.test.ts | 8 +- packages/stack/src/services/ProcessRecipe.ts | 48 +- .../stack/tests/artifact-store-fixture.ts | 93 ++ 21 files changed, 1722 insertions(+), 1095 deletions(-) create mode 100644 packages/stack/src/namespace/Pin.ts create mode 100644 packages/stack/src/namespace/drivers/sqlite-pin.ts create mode 100644 packages/stack/tests/artifact-store-fixture.ts diff --git a/apps/cli/src/command-internal/bundled-postgres-client.ts b/apps/cli/src/command-internal/bundled-postgres-client.ts index 8495a41661..b57ef39779 100644 --- a/apps/cli/src/command-internal/bundled-postgres-client.ts +++ b/apps/cli/src/command-internal/bundled-postgres-client.ts @@ -15,8 +15,8 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { ArtifactError, postgresVersion, - prepareNativeArtifact, resolveArtifact, + useNativeArtifact, } from "@supabase/stack/internal/artifacts"; import { DockerRun, type DockerRunOpts } from "./docker-run.service.ts"; @@ -249,7 +249,9 @@ export const bundledPostgresClientLayer: Layer.Layer< ); return yield* dockerRun(options, resolved.image, docker); } - const prepared = yield* prepareNativeArtifact( + // Pins the generation for the life of this scope, before the spawn inside `nativeRun`: + // the client child always exits before this scope closes, so the pin covers it fully. + const prepared = yield* useNativeArtifact( { service: "database", version }, path.join(settings.supabaseHome, "cache", "stack"), { os: runtimeInfo.platform, arch: runtimeInfo.arch }, diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index e8e7ec3383..d9f6f2543a 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -34,8 +34,9 @@ require template serving and shared external JWKS verification respectively. Secrets needed by enabled services are passed to the runtime. State and service data live under `$SUPABASE_HOME/stacks//` (`~/.supabase/stacks//` by default); native artifacts -use `$SUPABASE_HOME/cache/stack`. Storage files use the caller-owned project directory -`supabase/.temp/stack-uploads//`. Functions preparation may build the project's source. +use `$SUPABASE_HOME/cache/stack`, keyed by content digest and retired automatically once unused for +30 days. Storage files use the caller-owned project directory `supabase/.temp/stack-uploads//`. +Functions preparation may build the project's source. For a new stack, `--runtime auto` selects Docker when `docker version` reaches its daemon, then native on Linux x64/arm64 and macOS arm64. The probe is bounded by 10 seconds. Without a reachable @@ -50,9 +51,10 @@ Docker, and `--runtime native` for a new stack on platforms that support native. Native startup refuses root because PostgreSQL `initdb` cannot run as root, unless a Claude Code or Modal Sandbox is detected or `SUPABASE_NATIVE_POSTGRES_USER` names a non-root user. PostgreSQL then runs as that user: the CLI chowns the instance data, root key, socket directory, and the cached bundle's -`pgsodium_getkey.sh` to it, and adds traverse-only `o+x` to their parent directories, including -root's home directory. Later commands that restrict the artifact cache and stack state roots to -their owner keep that grant. +`pgsodium_getkey.sh` to it, and adds traverse-only `o+x` to the parent directories outside the +cached artifact (the artifact cache and stack state roots, including root's home directory), which +stay owner-restricted between launches otherwise. A directory inside the cached artifact itself is +never chmodded; it keeps the archive's own mode, and startup fails if one is not already traversable. Database is eager by default. Other services are lazy; traffic wakes them through their listeners. Lazy services with idle policies stop after 60 seconds without traffic, Studio after 5 minutes. A diff --git a/packages/stack/README.md b/packages/stack/README.md index e7f266be73..df18ace9ac 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -85,7 +85,9 @@ Omitted database `jwtSecret` and `rootKey` inputs use the shared local-developme as `DEFAULT_LOCAL_JWT_SECRET` and `DEFAULT_POSTGRES_ROOT_KEY`. Explicit values override these defaults. The effective root key is supplied through a stack-owned file for both native and container runtimes. -Native PostgreSQL refuses to run as uid 0. When the stack runs as root inside a detected agent sandbox (Claude Code, Modal Sandbox), or `SUPABASE_NATIVE_POSTGRES_USER=` names a non-root system user, only the PostgreSQL process runs as that user: the instance data directory, root key file, socket directory, and the bundle's `pgsodium_getkey.sh` are chowned to it, and the instance directory, the PostgreSQL bundle directory, and their ancestors receive traverse-only (`o+x`) permission, which the artifact cache and stack state keep when they restrict their roots to the owner. Running as root elsewhere fails before PostgreSQL launches. +The native artifact cache holds one immutable, content-addressed generation per published archive at `cacheRoot///`, published by renaming a lock-guarded `cacheRoot//.staging//content` into place. A generation is never modified after publish. `Artifacts.use` is the one scoped launch-time operation: it takes a SHARED lock on the generation's `cacheRoot//.lock` file before resolving or preparing it, and every consumer (service launch, startup commands, the PostgreSQL handover, one-shot commands, and the native launcher itself) uses the returned paths only inside that scope. A background sweep at prepare time retires a generation only once it is both unlocked and untouched for 30 days; digest lock files are never deleted. A stale, lock-guarded staging directory left by a killed preparer is reaped by the next preparer or sweep, never by age. + +Native PostgreSQL refuses to run as uid 0. When the stack runs as root inside a detected agent sandbox (Claude Code, Modal Sandbox), or `SUPABASE_NATIVE_POSTGRES_USER=` names a non-root system user, only the PostgreSQL process runs as that user: the instance data directory, root key file, socket directory, and the bundle's `pgsodium_getkey.sh` are chowned to it, and the instance directory and its ancestors outside the generation (the artifact cache and stack state roots, which otherwise stay restricted to their owner) receive traverse-only (`o+x`) permission. A directory inside the generation itself is never chmodded; it must already carry that bit from the archive, or the handover fails naming the path. The `pgsodium_getkey.sh` chown is the one accepted exception to generation immutability: the packaged init script hard-codes that cached path and `chmod +x`s it, which only its owner may do. Running as root elsewhere fails before PostgreSQL launches. Native PostgreSQL listens only on its socket and reads a stack-generated HBA file, written to the socket directory on every launch, instead of `PGDATA/pg_hba.conf`. It trusts `supabase_admin`, including through the proxied loopback database port, and requires `scram-sha-256` passwords from every other role. diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index acdf655490..d57d02560e 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -75,6 +75,8 @@ export interface PreparedNativeArtifact { readonly version: string; readonly root: string; readonly executable: string; + /** The generation's digest lock file; a native workload spawned from `root` pins this too. */ + readonly lockPath: string; } interface ArtifactDefinition { @@ -594,6 +596,49 @@ export const catalogPins = (): ReadonlyArray<{ const artifactKey = (artifact: SlimServicesArtifact): string => `slim-services/${artifact.service}/${artifact.version}/${artifact.target}`; +/** Builds the native store request shared by ahead-of-time preparation and launch-time use. */ +const nativeStoreRequest = Effect.fn("Artifacts.nativeStoreRequest")(function* ( + request: { readonly service: ServiceKind; readonly version?: string }, + platform: { readonly os: string; readonly arch: string }, +) { + const resolved = yield* resolveArtifact(request); + const target = targetForPlatform(platform); + if (target === undefined) + return yield* new ArtifactError({ + message: `Native artifacts are unsupported on ${platformText(platform)}`, + service: request.service, + version: resolved.version, + platform: platformText(platform), + }); + const sourceArtifact = artifactFor(request.service, resolved, target); + const key = artifactKey(sourceArtifact); + const source = makeSlimServicesSource((candidate) => + candidate.key === key ? sourceArtifact : undefined, + ); + return { + resolved, + storeRequest: { + key, + requiredRuntimePaths: resolved.requiredRuntimePaths, + executablePath: resolved.executablePath, + }, + source, + }; +}); + +const toNativeArtifact = ( + resolved: ArtifactResolution, + prepared: { readonly path: string; readonly lockPath: string }, + path: Path.Path, +): PreparedNativeArtifact => ({ + service: resolved.service, + version: resolved.version, + root: prepared.path, + executable: path.join(prepared.path, resolved.executablePath), + lockPath: prepared.lockPath, +}); + +/** Ahead-of-time: downloads and publishes the generation, but pins nothing. */ export const prepareNativeArtifact = Effect.fn("Artifacts.prepareNativeArtifact")(function* ( request: { readonly service: ServiceKind; readonly version?: string }, cacheRoot: string, @@ -604,32 +649,44 @@ export const prepareNativeArtifact = Effect.fn("Artifacts.prepareNativeArtifact" ) { const resolved = yield* resolveArtifact(request); return yield* Effect.gen(function* () { - const target = targetForPlatform(platform); - if (target === undefined) - return yield* new ArtifactError({ - message: `Native artifacts are unsupported on ${platformText(platform)}`, - service: request.service, - version: resolved.version, - platform: platformText(platform), - }); - const sourceArtifact = artifactFor(request.service, resolved, target); - const key = artifactKey(sourceArtifact); - const source = makeSlimServicesSource((candidate) => - candidate.key === key ? sourceArtifact : undefined, - ); - const store = yield* makeArtifactStore({ cacheRoot, source }); - const prepared = yield* store.prepare({ - key, - requiredRuntimePaths: resolved.requiredRuntimePaths, - executablePath: resolved.executablePath, - }); + const built = yield* nativeStoreRequest(request, platform); + const store = yield* makeArtifactStore({ cacheRoot, source: built.source }); + const prepared = yield* store.prepare(built.storeRequest); const path = yield* Path.Path; - return { - service: resolved.service, - version: resolved.version, - root: prepared.path, - executable: path.join(prepared.path, resolved.executablePath), - }; + return toNativeArtifact(built.resolved, prepared, path); + }).pipe( + Effect.mapError((cause) => + cause instanceof ArtifactError + ? cause + : new ArtifactError({ + message: `Unable to prepare ${request.service} artifact: ${errorMessage(cause)}`, + service: request.service, + version: resolved.version, + cause, + }), + ), + ); +}); + +/** + * The one scoped launch-time operation: pins the generation, resolves or prepares it, and returns + * its paths. Every consumer must use the returned paths only inside this scope. + */ +export const useNativeArtifact = Effect.fn("Artifacts.useNativeArtifact")(function* ( + request: { readonly service: ServiceKind; readonly version?: string }, + cacheRoot: string, + platform: { readonly os: string; readonly arch: string } = { + os: process.platform, + arch: process.arch, + }, +) { + const resolved = yield* resolveArtifact(request); + return yield* Effect.gen(function* () { + const built = yield* nativeStoreRequest(request, platform); + const store = yield* makeArtifactStore({ cacheRoot, source: built.source }); + const prepared = yield* store.use(built.storeRequest); + const path = yield* Path.Path; + return toNativeArtifact(built.resolved, prepared, path); }).pipe( Effect.mapError((cause) => cause instanceof ArtifactError diff --git a/packages/stack/src/host/CommandRunner.initialization.integration.test.ts b/packages/stack/src/host/CommandRunner.initialization.integration.test.ts index f5420bf5ee..8aa2bccadb 100644 --- a/packages/stack/src/host/CommandRunner.initialization.integration.test.ts +++ b/packages/stack/src/host/CommandRunner.initialization.integration.test.ts @@ -34,14 +34,18 @@ if (authVersion === undefined) { const prepareAuthArtifact = Effect.fn(function* (cacheRoot: string, script: string) { if (target === undefined) return yield* Effect.fail("Unsupported test platform"); const fs = yield* FileSystem.FileSystem; - const { releaseVersion } = yield* resolveArtifact({ service: "auth" }); + const { releaseVersion, natives } = yield* resolveArtifact({ service: "auth" }); const request: ArtifactRequest = { key: `slim-services/auth/${releaseVersion}/${target}`, requiredRuntimePaths: ["bin/auth"], executablePath: "bin/auth", }; + // The generation directory is content-addressed by this digest: it must match what + // `CommandRunner`'s own real artifact source resolves for this platform, or the fixture + // published here is invisible to it and it downloads the real catalog-pinned binary instead. + const digest = natives[target].archive; const source: ArtifactSource = { - checksum: () => Effect.succeed("0".repeat(64)), + checksum: () => Effect.succeed(digest), materialize: (_request, destination) => Effect.gen(function* () { yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); diff --git a/packages/stack/src/host/CommandRunner.ts b/packages/stack/src/host/CommandRunner.ts index 4074bf4b0b..5308b7d5be 100644 --- a/packages/stack/src/host/CommandRunner.ts +++ b/packages/stack/src/host/CommandRunner.ts @@ -17,9 +17,9 @@ import { ChildProcessSpawner } from "effect/unstable/process"; import { HttpClient } from "effect/unstable/http"; import { slimImageMirrors, - prepareNativeArtifact, postgresVersion, resolveArtifact, + useNativeArtifact, } from "../Artifacts.ts"; import type * as Claims from "../namespace/Claims.ts"; import * as Environment from "../namespace/Environment.ts"; @@ -184,7 +184,10 @@ const makeCommandRunner = (options: { initialization?.version ?? postgresVersion(String(postgresCommand?.command.major)); const process = yield* Effect.gen(function* () { if (container === undefined) { - const artifact = yield* prepareNativeArtifact( + // Pins the generation for the life of this job's scope, before the spawn below: the + // pin is registered ahead of the process, so it releases only after the process's + // own cleanup finalizer (added later) has run. + const artifact = yield* useNativeArtifact( { service: initialization?.service ?? "database", version }, options.cacheRoot, ).pipe( @@ -209,6 +212,7 @@ const makeCommandRunner = (options: { args: postgresCommand?.args ?? initialization?.args ?? [], env: postgresCommand?.env ?? initialization?.env ?? {}, environment, + artifactLockPath: artifact.lockPath, cwd: postgresCommand?.pgProve?.cwd ?? initialization?.cwd ?? diff --git a/packages/stack/src/internal/artifacts.ts b/packages/stack/src/internal/artifacts.ts index 0ee05f1fac..993bc1605e 100644 --- a/packages/stack/src/internal/artifacts.ts +++ b/packages/stack/src/internal/artifacts.ts @@ -7,6 +7,7 @@ export { postgresVersion, prepareNativeArtifact, resolveArtifact, + useNativeArtifact, type ArtifactPin, type NativePin, type ServiceKind, diff --git a/packages/stack/src/namespace/Lease.ts b/packages/stack/src/namespace/Lease.ts index 9ec17a6d6e..dbed876d42 100644 --- a/packages/stack/src/namespace/Lease.ts +++ b/packages/stack/src/namespace/Lease.ts @@ -8,7 +8,7 @@ import { } from "./Capabilities.ts"; import * as Publication from "./Publication.ts"; import { removeEmptyDirectory } from "./drivers/FileSystem.ts"; -import { acquireLock, errcode, isBusy, takeLock } from "./drivers/Sqlite.ts"; +import { acquireLock, errcode, isBusy, isMissing, takeLock } from "./drivers/Sqlite.ts"; /** What the current lease holder publishes: an owner's control endpoint, or a sweeper's marker. */ export const LeaseHolder = Schema.Union([ @@ -86,7 +86,6 @@ export const make = ( const retryTransientRead = (effect: Effect.Effect) => effect.pipe(Effect.retry({ schedule: transientRetrySchedule, while: sharingViolation })); - const isMissing = (error: NamespaceError) => errcode(error.cause, 14); /** An open file that was unlinked: SQLite IOERR_VNODE on macOS, IOERR_FSTAT on Linux. */ const isMoved = (error: NamespaceError) => errcode(error.cause, 6922) || errcode(error.cause, 1802); diff --git a/packages/stack/src/namespace/Pin.ts b/packages/stack/src/namespace/Pin.ts new file mode 100644 index 0000000000..270bb88035 --- /dev/null +++ b/packages/stack/src/namespace/Pin.ts @@ -0,0 +1,80 @@ +import { DateTime, Effect, Exit, FileSystem, Schedule, Scope, Semaphore } from "effect"; +import { namespaceError, type NamespaceError } from "./Capabilities.ts"; +import { acquireLock, isBusy, takeSharedLock } from "./drivers/Sqlite.ts"; + +/** + * Open reader connections this process currently holds, keyed by canonical digest lock path. A + * second `acquireLock` on the same path would either be rejected by its own single-holder guard or + * (worse) silently coalesce, since POSIX advisory locks belong to a process, not a descriptor. So + * every local pin on the same generation shares this one connection, reference-counted, instead of + * opening a second one; this is also the registry a retirement sweep's own `acquireLock` attempt + * naturally defers to, since that attempt targets the same path and the same single-holder guard. + */ +const registry = new Map(); + +/** + * Serializes the check-or-create section per path: checking `registry` and opening its connection + * span several `yield*` suspension points, so two concurrent local pins on a brand new path would + * otherwise both see it missing and race to open it. One semaphore per path, created and looked up + * synchronously (never itself awaited to exist), rules that out. + */ +const locks = new Map(); +const semaphoreFor = (path: string): Semaphore.Semaphore => { + const existing = locks.get(path); + if (existing !== undefined) return existing; + const created = Semaphore.makeUnsafe(1); + locks.set(path, created); + return created; +}; + +/** Rides out a retirement sweep's brief EXCLUSIVE hold instead of failing a concurrent pin. */ +const pinRetrySchedule = Schedule.spaced("10 millis").pipe( + Schedule.upTo({ duration: "2 seconds" }), +); + +/** + * Holds a SHARED lock on `lockPath` (one generation's digest lock file) for the life of the + * returned scope: every consumer of a prepared generation's paths must stay inside this scope, and + * must not resolve or create `lockPath`'s generation directory before this resolves. Touches the + * lock file's mtime on first acquisition in this process, which is retirement's only liveness + * signal for a holder in another process. The caller must ensure `lockPath`'s parent directory + * already exists. + */ +export const pin = ( + lockPath: string, +): Effect.Effect => + Effect.acquireRelease( + semaphoreFor(lockPath).withPermits(1)( + Effect.gen(function* () { + const existing = registry.get(lockPath); + if (existing !== undefined) { + existing.refCount++; + return; + } + const scope = yield* Scope.make(); + yield* Effect.gen(function* () { + const connection = yield* acquireLock(lockPath, "create").pipe(Scope.provide(scope)); + yield* takeSharedLock(connection).pipe( + Effect.retry({ schedule: pinRetrySchedule, while: isBusy }), + ); + const fs = yield* FileSystem.FileSystem; + const now = DateTime.toDateUtc(yield* DateTime.now); + yield* fs + .utimes(lockPath, now, now) + .pipe(Effect.mapError((cause) => namespaceError("pin-touch", cause))); + }).pipe(Effect.tapError(() => Scope.close(scope, Exit.void))); + registry.set(lockPath, { scope, refCount: 1 }); + }), + ), + () => + semaphoreFor(lockPath).withPermits(1)( + Effect.suspend(() => { + const entry = registry.get(lockPath); + if (entry === undefined) return Effect.void; + entry.refCount--; + if (entry.refCount > 0) return Effect.void; + registry.delete(lockPath); + return Scope.close(entry.scope, Exit.void); + }), + ), + ); diff --git a/packages/stack/src/namespace/drivers/Sqlite.ts b/packages/stack/src/namespace/drivers/Sqlite.ts index 2280a3825b..a0f8bd0a98 100644 --- a/packages/stack/src/namespace/drivers/Sqlite.ts +++ b/packages/stack/src/namespace/drivers/Sqlite.ts @@ -3,6 +3,7 @@ import { DatabaseSync } from "node:sqlite"; import { pathToFileURL } from "node:url"; import { Effect, Predicate, Scope } from "effect"; import { namespaceError, type NamespaceError } from "../Capabilities.ts"; +import { takeSharedLockSync } from "./sqlite-pin.ts"; const openConnection = (path: string, mode: "create" | "existing") => Effect.try({ @@ -14,7 +15,17 @@ const openConnection = (path: string, mode: "create" | "existing") => }); const closeConnection = (connection: DatabaseSync) => Effect.try({ - try: () => connection.close(), + // Rolls back first: closing a connection mid-transaction must release its lock immediately, + // and must not depend on `close()` alone to do so. A connection with no open transaction (for + // example `acquireLock("create")` with no lock ever taken) has nothing to roll back. + try: () => { + try { + connection.exec("ROLLBACK"); + } catch { + // No open transaction; nothing to roll back. + } + connection.close(); + }, catch: (cause) => namespaceError("lock-close", cause), }); @@ -23,6 +34,8 @@ export const errcode = (cause: unknown, code: number) => Predicate.hasProperty(cause, "errcode") && cause.errcode === code; /** SQLite reports lock contention under `busy_timeout = 0` with this errcode. */ export const isBusy = (error: NamespaceError) => errcode(error.cause, 5); +/** `acquireLock("existing")` reports this errcode when the database file does not exist. */ +export const isMissing = (error: NamespaceError) => errcode(error.cause, 14); const busy = (operation: string) => namespaceError(operation, Object.assign(new Error("locked by this process"), { errcode: 5 })); @@ -59,7 +72,12 @@ export const acquireLock = ( ); }); -/** Takes the connection's write lock once; fails busy under any contention instead of waiting. */ +/** + * Takes the connection's RESERVED write lock once; fails busy under any contention instead of + * waiting. `BEGIN IMMEDIATE` alone excludes other writers, but not an existing reader's SHARED + * lock (it only escalates to EXCLUSIVE, conflicting with readers, when a write is attempted), so + * this is enough for writer-versus-writer exclusion only, such as one staging slot's own lock. + */ export const takeLock = (connection: DatabaseSync): Effect.Effect => Effect.gen(function* () { yield* Effect.try({ @@ -71,3 +89,30 @@ export const takeLock = (connection: DatabaseSync): Effect.Effect namespaceError("lock", cause), }); }); + +/** + * Takes the connection's EXCLUSIVE lock once, immediately, in rollback-journal mode: unlike + * `BEGIN IMMEDIATE`, this conflicts with an existing reader's SHARED lock right away, which is + * what retirement needs to safely exclude a live pin. Fails busy under any contention. + */ +export const takeExclusiveLock = (connection: DatabaseSync): Effect.Effect => + Effect.gen(function* () { + yield* Effect.try({ + try: () => connection.exec("PRAGMA busy_timeout = 0"), + catch: (cause) => namespaceError("lock", cause), + }); + yield* Effect.try({ + try: () => connection.exec("BEGIN EXCLUSIVE"), + catch: (cause) => namespaceError("lock", cause), + }); + }); + +/** + * Takes the connection's SHARED read lock once: a real `SELECT` against a lazily created one-row + * table, held until the connection commits or closes. Busy under a concurrent EXCLUSIVE holder. + */ +export const takeSharedLock = (connection: DatabaseSync): Effect.Effect => + Effect.try({ + try: () => takeSharedLockSync(connection), + catch: (cause) => namespaceError("lock", cause), + }); diff --git a/packages/stack/src/namespace/drivers/sqlite-pin.ts b/packages/stack/src/namespace/drivers/sqlite-pin.ts new file mode 100644 index 0000000000..499d8e3ee9 --- /dev/null +++ b/packages/stack/src/namespace/drivers/sqlite-pin.ts @@ -0,0 +1,39 @@ +// Framework-agnostic: shared by the Effect-wrapped driver (`Sqlite.ts`, used in-process) and the +// standalone native launcher, which runs outside any Effect runtime and calls these directly. +import type { DatabaseSync } from "node:sqlite"; + +/** + * The one-row table a pin's `SELECT` reads, created lazily so a fresh digest lock file works. + * Checks first and only writes when actually missing: every pin after the first ever one on a + * digest finds the table and row already there, so it never takes a write-intent lock at all, + * which would otherwise transiently conflict with another concurrent reader's own no-op check. + */ +export const ensurePinTable = (connection: DatabaseSync): void => { + const hasTable = connection + .prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'pin'") + .get(); + if (hasTable === undefined) + connection.exec("CREATE TABLE IF NOT EXISTS pin (id INTEGER PRIMARY KEY)"); + const hasRow = connection.prepare("SELECT 1 FROM pin LIMIT 1").get(); + if (hasRow === undefined) connection.exec("INSERT OR IGNORE INTO pin (id) VALUES (1)"); +}; + +/** + * Takes a SHARED read lock: a deferred `BEGIN` plus a real `SELECT`, held until the connection + * commits or closes. Multiple SHARED holders coexist; only a concurrent EXCLUSIVE holder (a + * retirement sweep) contends, surfacing as SQLite's busy errcode under `busy_timeout = 0`. + */ +export const takeSharedLockSync = (connection: DatabaseSync): void => { + connection.exec("PRAGMA busy_timeout = 0"); + ensurePinTable(connection); + connection.exec("BEGIN"); + try { + connection.prepare("SELECT id FROM pin LIMIT 1").get(); + } catch (error) { + // A deferred `BEGIN` takes no lock by itself; only this `SELECT` does, so a failure here + // still leaves the transaction open. Roll back before propagating, or a retry on this same + // connection would hit "cannot start a transaction within a transaction" instead of busy. + connection.exec("ROLLBACK"); + throw error; + } +}; diff --git a/packages/stack/src/preparation/ArtifactStore.ts b/packages/stack/src/preparation/ArtifactStore.ts index 1b21d3c92e..e4ff7504cd 100644 --- a/packages/stack/src/preparation/ArtifactStore.ts +++ b/packages/stack/src/preparation/ArtifactStore.ts @@ -1,19 +1,17 @@ -import { - Clock, - Crypto, - Effect, - FileSystem, - Option, - Path, - PlatformError, - Predicate, - Schema, -} from "effect"; +import { Clock, Crypto, Effect, FileSystem, Option, Path, PlatformError, Predicate } from "effect"; import { HttpClient } from "effect/unstable/http"; import { ChildProcessSpawner } from "effect/unstable/process"; import { ArtifactIntegrityError, PreparationError } from "./Errors.ts"; import { validateRelativePath, validateSha256 } from "./Integrity.ts"; import { restrictDirectoryToOwner } from "../runtime/postgres-user.ts"; +import { fsyncDirectory } from "../namespace/drivers/FileSystem.ts"; +import { + acquireLock, + isMissing, + takeExclusiveLock, + takeLock, +} from "../namespace/drivers/Sqlite.ts"; +import * as Pin from "../namespace/Pin.ts"; /** A concrete artifact identity. `key` may contain subdirectories but never an absolute or traversing path. */ export interface ArtifactRequest { @@ -56,10 +54,12 @@ export interface ArtifactStoreOptions { readonly source: ArtifactSource; } -interface PreparedArtifact { +export interface PreparedArtifact { readonly key: string; - /** Installed artifact directory. Required runtime paths are relative to this directory. */ + /** Installed, content-addressed generation directory. Required runtime paths are relative to it. */ readonly path: string; + /** The generation's digest lock file; pinned by `use` for the life of its returned scope. */ + readonly lockPath: string; readonly sha256: string; readonly requiredRuntimePaths: ReadonlyArray; readonly executablePath?: string; @@ -68,14 +68,14 @@ interface PreparedArtifact { type ArtifactStoreError = PreparationError | ArtifactIntegrityError; -const ARTIFACT_FORMAT = "supabase-stack-artifact-v3"; -const METADATA_NAME = ".artifact.json"; -const EXECUTABLE_MODE = 0o755; -/** - * Child of the staging directory that sources materialize into: GNU tar resets its extraction - * target's mtime from the archive, which must not make a staging directory look like a leftover. - */ +/** Child of a staging directory that sources materialize into, renamed whole to publish. */ const STAGING_CONTENT_NAME = "content"; +/** Per-key directory holding every in-progress preparer's and retirer's staging slot. */ +const STAGING_DIR_NAME = ".staging"; +const DIRECTORY_MODE = 0o755; +/** A generation is eligible for retirement once its digest lock file's mtime is this old. */ +const RETIREMENT_AGE_MILLIS = 30 * 24 * 60 * 60 * 1000; +const DIGEST_NAME = /^[0-9a-f]{64}$/u; type ArtifactPathKind = "file" | "directory" | "symlink"; @@ -91,19 +91,6 @@ const artifactError = (message: string, fields: Readonly const metadataError = (message: string, fields: Readonly> = {}) => new ArtifactIntegrityError({ ...fields, message }); -const ArtifactMetadataSchema = Schema.Struct({ - format: Schema.Literal(ARTIFACT_FORMAT), - key: Schema.String, - sha256: Schema.String, - requiredRuntimePaths: Schema.Array(Schema.String), - requiredRuntimeKinds: Schema.Record( - Schema.String, - Schema.Literals(["file", "directory", "symlink"]), - ), - executablePath: Schema.optional(Schema.String), -}); -type ArtifactMetadata = Schema.Schema.Type; - const mapFs = ( path: string, operation: string, @@ -163,47 +150,11 @@ const validateRequest = (request: ArtifactRequest): Effect.Effect>, -): ArtifactMetadata => ({ - format: ARTIFACT_FORMAT, - key: request.key, - sha256, - requiredRuntimePaths: [...request.requiredRuntimePaths], - requiredRuntimeKinds, - ...(request.executablePath === undefined ? {} : { executablePath: request.executablePath }), -}); - -const encodeMetadata = (metadata: ArtifactMetadata): Effect.Effect => - Schema.encodeEffect(Schema.fromJsonString(ArtifactMetadataSchema))(metadata).pipe( - Effect.mapError((cause) => - artifactError(`Unable to encode artifact metadata: ${String(cause)}`), - ), - ); - -const readMetadata = ( - fs: FileSystem.FileSystem, - metadataPath: string, -): Effect.Effect, ArtifactIntegrityError> => - Effect.matchEffect(fs.readFileString(metadataPath), { - onFailure: (cause) => - isNotFound(cause) - ? Effect.succeed(Option.none()) - : Effect.fail( - metadataError("Cached artifact metadata is unreadable", { - path: metadataPath, - cause, - }), - ), - onSuccess: (text) => - Schema.decodeEffect(Schema.fromJsonString(ArtifactMetadataSchema))(text).pipe( - Effect.map(Option.some), - Effect.orElseSucceed(() => Option.none()), - ), - }); - +/** + * Creates `directory` (and any missing ancestors up to `canonicalRoot`) and restricts every + * segment along the way to its owner, preserving a traverse-only grant: every container level + * under the cache root, not just the leaf, must stay owner-restricted as the tree grows. + */ const ensureDirectory = ( fs: FileSystem.FileSystem, path: Path.Path, @@ -220,7 +171,14 @@ const ensureDirectory = ( for (const segment of relative.split(path.sep).filter((value) => value.length > 0)) { current = path.join(current, segment); const exists = yield* mapFs(current, "inspect artifact directory", fs.exists(current)); - if (!exists) break; + if (!exists) + // `recursive: true` tolerates a concurrent preparer that just created this very segment + // between the `exists` check above and this call; it is otherwise a single-directory make. + yield* mapFs( + current, + "create artifact directory", + fs.makeDirectory(current, { recursive: true, mode: 0o700 }), + ); const real = yield* fs.realPath(current).pipe( Effect.mapError((cause) => artifactError(`Unable to resolve artifact directory: ${cause.message}`, { @@ -233,25 +191,8 @@ const ensureDirectory = ( return yield* artifactError("Artifact directory contains a symlink", { path: current }); if (!pathAtOrBelow(root, real, path.sep)) return yield* artifactError("Artifact directory escapes cache root", { path: current }); + yield* mapFs(current, "secure artifact directory", restrictDirectoryToOwner(fs, current)); } - yield* mapFs( - resolved, - "create artifact directory", - fs.makeDirectory(resolved, { recursive: true, mode: 0o700 }), - ); - const real = yield* fs.realPath(resolved).pipe( - Effect.mapError((cause) => - artifactError(`Unable to resolve artifact directory: ${cause.message}`, { - path: resolved, - cause, - }), - ), - ); - if (real !== resolved) - return yield* artifactError("Artifact directory contains a symlink", { path: resolved }); - if (!pathAtOrBelow(root, real, path.sep)) - return yield* artifactError("Artifact directory escapes cache root", { path: resolved }); - yield* mapFs(resolved, "secure artifact directory", restrictDirectoryToOwner(fs, resolved)); }); const ensureSafeRoot = ( @@ -447,12 +388,12 @@ const validateFreshRuntimePaths = ( return Object.fromEntries(entries); }); -const ensureSafePaths = ( +/** Shallow containment+kind check for an already-published, trusted generation: no recursion. */ +const inspectRequiredPaths = ( fs: FileSystem.FileSystem, path: Path.Path, root: string, realRoot: string, - metadata: ArtifactMetadata, relativePaths: ReadonlyArray, ): Effect.Effect>, ArtifactIntegrityError> => Effect.gen(function* () { @@ -475,23 +416,17 @@ const ensureSafePaths = ( path: relative, }); const inspected = yield* inspectFreshPath(fs, path, candidate, realRoot); - const expectedKind = metadata.requiredRuntimeKinds[relative]; - if (expectedKind === undefined || inspected.kind !== expectedKind) - return yield* metadataError("Cached artifact runtime path changed basic kind", { - path: relative, - expected: expectedKind, - actual: inspected.kind, - }); entries.push([relative, inspected]); } return Object.fromEntries(entries); }); +/** Kind-only executable check, used before a fresh publish normalizes its final mode. */ const ensureExecutableFile = ( fs: FileSystem.FileSystem, executable: string, -): Effect.Effect => { - return fs.stat(executable).pipe( +): Effect.Effect => + fs.stat(executable).pipe( Effect.mapError((cause) => metadataError("Cached artifact executable cannot be inspected", { path: executable, @@ -509,73 +444,30 @@ const ensureExecutableFile = ( ), ), ); -}; - -const identityMismatch = (request: ArtifactRequest, metadata: ArtifactMetadata): boolean => - metadata.key !== request.key || metadata.executablePath !== request.executablePath; - -const unrecordedRequiredPaths = ( - request: ArtifactRequest, - metadata: ArtifactMetadata, -): ReadonlyArray => - request.requiredRuntimePaths.filter( - (relative) => metadata.requiredRuntimeKinds[relative] === undefined, - ); -const sha256Of = ( - request: ArtifactRequest, - metadata: ArtifactMetadata, -): Effect.Effect => - validateSha256(metadata.sha256).pipe( +/** A published generation is never chmodded again, so a cache hit must check the bit directly. */ +const ensureExecutableMode = ( + fs: FileSystem.FileSystem, + executable: string, +): Effect.Effect => + fs.stat(executable).pipe( Effect.mapError((cause) => - metadataError("Cached artifact metadata contains an invalid SHA-256", { - key: request.key, + metadataError("Cached artifact executable cannot be inspected", { + path: executable, cause, }), ), + Effect.flatMap((info) => + (info.mode & 0o100) !== 0 + ? Effect.void + : Effect.fail( + metadataError("Cached artifact executable is missing its executable bit", { + path: executable, + }), + ), + ), ); -const writeBytesSync = ( - fs: FileSystem.FileSystem, - path: string, - bytes: Uint8Array, -): Effect.Effect => - Effect.scoped( - Effect.gen(function* () { - const file = yield* fs.open(path, { flag: "wx", mode: 0o600 }).pipe( - Effect.mapError((cause) => - artifactError(`Unable to create artifact source file: ${cause.message}`, { - path, - cause, - }), - ), - ); - yield* file.writeAll(bytes).pipe( - Effect.mapError((cause) => - artifactError(`Unable to write artifact source file: ${cause.message}`, { - path, - cause, - }), - ), - ); - yield* file.sync.pipe( - Effect.mapError((cause) => - artifactError(`Unable to sync artifact source file: ${cause.message}`, { path, cause }), - ), - ); - }), - ); - -const writeMetadataSync = ( - fs: FileSystem.FileSystem, - path: string, - metadata: ArtifactMetadata, -): Effect.Effect => - Effect.gen(function* () { - const encoded = yield* encodeMetadata(metadata); - yield* writeBytesSync(fs, path, new TextEncoder().encode(encoded)); - }); - const cleanup = (fs: FileSystem.FileSystem, path: string): Effect.Effect => fs .remove(path, { recursive: true, force: true }) @@ -585,85 +477,251 @@ const cleanup = (fs: FileSystem.FileSystem, path: string): Effect.Effect => - Effect.gen(function* () { - const token = yield* crypto.randomUUIDv4.pipe( - Effect.mapError((cause) => - artifactError(`Unable to allocate artifact metadata temporary name: ${cause.message}`, { - path: metadataPath, - cause, - }), + cacheRoot: string, + source: ArtifactSource, + request: ArtifactRequest, +): Effect.fn.Return { + const keyRoot = path.resolve(cacheRoot, request.key); + if (!pathWithin(cacheRoot, keyRoot, path.sep)) + return yield* artifactError("Artifact key escapes cache root", { key: request.key }); + const expectedSha256 = yield* source.checksum(request).pipe( + Effect.flatMap((sha256) => + validateSha256(sha256).pipe( + Effect.mapError((cause) => + metadataError("Artifact source returned an invalid SHA-256", { + key: request.key, + cause, + }), + ), ), - ); - const temporary = path.join(path.dirname(metadataPath), `${METADATA_NAME}.${token}.tmp`); - yield* Effect.gen(function* () { - yield* writeMetadataSync(fs, temporary, metadata); - yield* mapFs( - metadataPath, - "update cached artifact metadata", - fs.rename(temporary, metadataPath), - ); - }).pipe(Effect.onExit(() => cleanup(fs, temporary))); - }); + ), + ); + return { + keyRoot, + stagingRoot: path.join(keyRoot, STAGING_DIR_NAME), + expectedSha256, + generationPath: path.join(keyRoot, expectedSha256), + lockPath: path.join(keyRoot, `${expectedSha256}.lock`), + }; +}); -const orphanMaxAgeMillis = 24 * 60 * 60 * 1000; +const stagingDirFor = (path: Path.Path, stagingRoot: string, token: string) => + path.join(stagingRoot, token); +const stagingLockFor = (path: Path.Path, stagingRoot: string, token: string) => + path.join(stagingRoot, `${token}.lock`); -const leftoverToken = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u; +/** A staging lock's own file, or the rollback journal SQLite leaves while a transaction is open. */ +const STAGING_LOCK_SUFFIX = /\.lock(-journal)?$/u; -/** Matches the temporary and quarantine names the store creates beside `targetName`. */ -const isLeftoverOf = (targetName: string, name: string): boolean => { - const prefix = `.${targetName}.`; - const suffix = [".tmp", ".invalid"].find((candidate) => name.endsWith(candidate)); - return ( - suffix !== undefined && - name.startsWith(prefix) && - leftoverToken.test(name.slice(prefix.length, name.length - suffix.length)) +/** Every `.staging` token with a directory, a lock file, or both: a reap target either way. */ +const listStagingTokens = ( + fs: FileSystem.FileSystem, + stagingRoot: string, +): Effect.Effect> => + fs.readDirectory(stagingRoot).pipe( + Effect.map((names) => + Array.from(new Set(names.map((name) => name.replace(STAGING_LOCK_SUFFIX, "")))), + ), + Effect.orElseSucceed(() => []), ); -}; /** - * Best-effort removal of one target's temp/quarantine siblings that a hard kill left behind. - * Only entries older than `orphanMaxAgeMillis` are removed, so a concurrent in-progress download - * by another process is never touched. + * Removes a `.staging/` directory and its lock file only when the lock can be taken + * without blocking: a live preparer or retirer holding it keeps this from ever touching + * in-progress work. Best-effort: any failure (including contention) is silently skipped. */ -const reapLeftoversOf = ( +const reapStagingToken = ( + fs: FileSystem.FileSystem, + path: Path.Path, + stagingRoot: string, + token: string, +): Effect.Effect => { + const lockPath = stagingLockFor(path, stagingRoot, token); + const dir = stagingDirFor(path, stagingRoot, token); + return Effect.scoped( + Effect.gen(function* () { + const connection = yield* acquireLock(lockPath, "existing").pipe( + Effect.catchIf(isMissing, () => acquireLock(lockPath, "create")), + ); + yield* takeLock(connection); + yield* fs.remove(dir, { recursive: true, force: true }); + yield* fs.remove(lockPath, { force: true }); + yield* fs.remove(`${lockPath}-journal`, { force: true }); + }), + ).pipe(Effect.ignore); +}; + +const reapStaleStaging = ( fs: FileSystem.FileSystem, path: Path.Path, - target: string, + stagingRoot: string, ): Effect.Effect => + listStagingTokens(fs, stagingRoot).pipe( + Effect.flatMap((tokens) => + Effect.forEach(tokens, (token) => reapStagingToken(fs, path, stagingRoot, token), { + discard: true, + concurrency: 1, + }), + ), + ); + +/** Relocates a to-be-retired generation into a fresh, lock-guarded staging slot, then deletes it. */ +const relocateToStaging = Effect.fn("ArtifactStore.relocateToStaging")(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + crypto: Crypto.Crypto, + stagingRoot: string, + generationPath: string, +) { + yield* fs.makeDirectory(stagingRoot, { recursive: true, mode: 0o700 }); + const token = yield* crypto.randomUUIDv4; + const stagingLockPath = stagingLockFor(path, stagingRoot, token); + const stagingDir = stagingDirFor(path, stagingRoot, token); + yield* Effect.scoped( + Effect.gen(function* () { + const connection = yield* acquireLock(stagingLockPath, "create"); + yield* takeLock(connection); + yield* fs.rename(generationPath, stagingDir); + yield* fs.remove(stagingDir, { recursive: true, force: true }); + }), + ); + yield* fs + .remove(stagingLockPath, { force: true }) + .pipe(Effect.andThen(fs.remove(`${stagingLockPath}-journal`, { force: true })), Effect.ignore); +}); + +/** + * Recursively discovers every generation (a digest-named directory) anywhere under `root`, + * without ever descending into one: a key embeds its release version (see `Artifacts.ts`'s + * `artifactKey`), so a CLI upgrade leaves a previous release's generations behind a now-unreachable + * key that only a root-wide walk like this ever revisits. A plain directory-name listing, never a + * generation's own (potentially large) published contents, keeps this cheap. + */ +const discoverGenerations = ( + fs: FileSystem.FileSystem, + path: Path.Path, + root: string, +): Effect.Effect, never> => Effect.gen(function* () { - const now = yield* Clock.currentTimeMillis; - const parent = path.dirname(target); - const targetName = path.basename(target); - const names = yield* fs.readDirectory(parent); - yield* Effect.forEach( - names.filter((name) => isLeftoverOf(targetName, name)), - (name) => { - const candidate = path.join(parent, name); - return fs.stat(candidate).pipe( - Effect.flatMap((info) => - Option.match(info.mtime, { - onNone: () => Effect.void, - onSome: (mtime) => - now - mtime.getTime() > orphanMaxAgeMillis - ? fs.remove(candidate, { recursive: true, force: true }) - : Effect.void, - }), - ), - Effect.ignore, - ); - }, - { discard: true }, - ); - }).pipe(Effect.ignore); + const names = yield* fs.readDirectory(root).pipe(Effect.orElseSucceed(() => [])); + const found: Array<{ readonly keyRoot: string; readonly digest: string }> = []; + for (const name of names) { + if (name === STAGING_DIR_NAME) continue; + if (DIGEST_NAME.test(name)) { + found.push({ keyRoot: root, digest: name }); + continue; + } + found.push(...(yield* discoverGenerations(fs, path, path.join(root, name)))); + } + return found; + }); -const makeArtifactOperation = Effect.fn("ArtifactStore.operation")(function* ( +/** + * One best-effort sweep over every generation published anywhere under the cache root: a + * generation is retired only once an uncontended EXCLUSIVE lock on its digest lock file is taken + * AND, rechecked while holding that lock, its mtime is older than the retention window. Digest + * lock files are never deleted (a stable inode); only the generation directory they guard is. A + * per-digest failure (including contention from a live pin or another retirer) is skipped, never + * propagated: this sweep must never block the `prepare`/`use` call it runs alongside. + */ +const retireStaleGenerations = Effect.fn("ArtifactStore.retire")(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + crypto: Crypto.Crypto, + cacheRoot: string, +): Effect.fn.Return { + const generations = yield* discoverGenerations(fs, path, cacheRoot); + let retired = 0; + for (const { keyRoot, digest } of generations) { + const generationPath = path.join(keyRoot, digest); + const lockPath = path.join(keyRoot, `${digest}.lock`); + const stagingRoot = path.join(keyRoot, STAGING_DIR_NAME); + const didRetire = yield* Effect.scoped( + Effect.gen(function* () { + const connection = yield* acquireLock(lockPath, "existing"); + yield* takeExclusiveLock(connection); + const info = yield* fs.stat(lockPath); + const mtime = Option.getOrUndefined(info.mtime); + const now = yield* Clock.currentTimeMillis; + if (mtime === undefined || now - mtime.getTime() <= RETIREMENT_AGE_MILLIS) return false; + yield* relocateToStaging(fs, path, crypto, stagingRoot, generationPath); + return true; + }), + ).pipe(Effect.orElseSucceed(() => false)); + if (didRetire) retired++; + } + yield* Effect.annotateCurrentSpan({ "artifact.retired_count": retired }); + return retired; +}); + +/** A cache hit: the generation directory exists, so every listed path is checked, never rehashed. */ +const checkHit = Effect.fn("ArtifactStore.checkHit")(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + cacheRoot: string, + request: ArtifactRequest, + resolved: ResolvedGeneration, +): Effect.fn.Return, ArtifactStoreError> { + const exists = yield* mapFs( + resolved.generationPath, + "inspect cached artifact", + fs.exists(resolved.generationPath), + ); + if (!exists) return Option.none(); + const realRoot = yield* ensureSafeRoot(fs, path, resolved.generationPath, cacheRoot).pipe( + Effect.catch((error) => (isMissingArtifactRoot(error) ? Effect.void : Effect.fail(error))), + ); + if (realRoot === undefined) return Option.none(); + const inspected = yield* inspectRequiredPaths( + fs, + path, + resolved.generationPath, + realRoot, + request.requiredRuntimePaths, + ); + if (request.executablePath !== undefined) { + const executable = inspected[request.executablePath]; + if (executable === undefined) + return yield* metadataError("Cached artifact executable path is not recorded", { + path: request.executablePath, + }); + yield* ensureExecutableFile(fs, executable.realPath); + yield* ensureExecutableMode(fs, executable.realPath); + } + return Option.some({ + key: request.key, + path: resolved.generationPath, + lockPath: resolved.lockPath, + sha256: resolved.expectedSha256, + requiredRuntimePaths: [...request.requiredRuntimePaths], + ...(request.executablePath === undefined ? {} : { executablePath: request.executablePath }), + outcome: "cached" as const, + }); +}); + +const isPublishTargetTaken = (cause: unknown): boolean => + cause instanceof PlatformError.PlatformError && + cause.reason instanceof PlatformError.SystemError && + hasErrnoCode(cause.reason.cause) && + ["EEXIST", "ENOTEMPTY"].includes(String(cause.reason.cause.code)); + +/** + * Downloads, validates and publishes a fresh generation. Staged under a lock-guarded + * `.staging//content`, published by renaming the completed content to `/` and + * fsyncing the parent. A losing preparer (publish target already taken) accepts the winner's + * generation instead of failing, since both staged the same content-addressed digest. + */ +const publishGeneration = Effect.fn("ArtifactStore.publish")(function* ( fs: FileSystem.FileSystem, path: Path.Path, crypto: Crypto.Crypto, @@ -671,175 +729,46 @@ const makeArtifactOperation = Effect.fn("ArtifactStore.operation")(function* ( cacheRoot: string, source: ArtifactSource, request: ArtifactRequest, + resolved: ResolvedGeneration, onProgress?: (state: "downloading" | "preparing") => void, -) { - return yield* Effect.gen(function* () { - const target = path.resolve(cacheRoot, request.key); - const targetParent = path.dirname(target); - yield* ensureDirectory(fs, path, targetParent, cacheRoot); - yield* reapLeftoversOf(fs, path, target); - const metadataPath = path.join(target, METADATA_NAME); - const checkCached: Effect.Effect< - Option.Option, - ArtifactStoreError - > = Effect.gen(function* () { - const realRoot = yield* ensureSafeRoot(fs, path, target, cacheRoot).pipe( - Effect.map(Option.some), - Effect.catch((error) => - isMissingArtifactRoot(error) ? Effect.succeed(Option.none()) : Effect.fail(error), +): Effect.fn.Return { + yield* mapFs( + resolved.stagingRoot, + "create artifact staging root", + fs.makeDirectory(resolved.stagingRoot, { recursive: true, mode: 0o700 }), + ); + const token = yield* crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => + artifactError(`Unable to allocate artifact staging name: ${cause.message}`, { + key: request.key, + cause, + }), + ), + ); + const stagingDir = stagingDirFor(path, resolved.stagingRoot, token); + const stagingLockPath = stagingLockFor(path, resolved.stagingRoot, token); + const content = path.join(stagingDir, STAGING_CONTENT_NAME); + yield* Effect.scoped( + Effect.gen(function* () { + const connection = yield* acquireLock(stagingLockPath, "create").pipe( + Effect.mapError((cause) => + artifactError(`Unable to lock artifact staging directory: ${cause.message}`, { + key: request.key, + cause, + }), ), ); - if (Option.isNone(realRoot)) return Option.none(); - const cachedMetadata = yield* readMetadata(fs, metadataPath); - if (Option.isNone(cachedMetadata)) return Option.none(); - const metadata = cachedMetadata.value; - // A key names immutable content and sources unpack whole archives, so only a different key - // or executable is a miss; paths beyond the recorded ones are checked in the published tree. - if (identityMismatch(request, metadata)) return Option.none(); - const sha256 = yield* sha256Of(request, metadata); - const newPaths = unrecordedRequiredPaths(request, metadata); - const newPathSet = new Set(newPaths); - const recordedPaths = request.requiredRuntimePaths.filter( - (relative) => !newPathSet.has(relative), - ); - // Published content is not rehashed on cache hits: metadata and cheap structural checks - // protect the cache boundary; content tampering may execute or fail later when the - // workload starts. - const recordedSafePaths = yield* ensureSafePaths( - fs, - path, - target, - realRoot.value, - metadata, - recordedPaths, - ); - // A path the recorded metadata has not seen yet gets the same containment and safety - // validation a fresh publish applies, before its kind is trusted and recorded. - const newSafePaths = - newPaths.length > 0 - ? yield* validateFreshRuntimePaths(fs, path, target, realRoot.value, newPaths).pipe( - Effect.mapError((cause) => - metadataError( - `Cached artifact ${request.key} cannot serve newly required runtime paths (${cause.message}); remove ${target} to download it again`, - { key: request.key, path: target, cause }, - ), - ), - ) - : {}; - const safePaths = { ...recordedSafePaths, ...newSafePaths }; - if (newPaths.length > 0) { - const newKinds = Object.fromEntries( - Object.entries(newSafePaths).map(([relative, inspected]) => [relative, inspected.kind]), - ); - const appendedPaths = newPaths.filter( - (relative) => !metadata.requiredRuntimePaths.includes(relative), - ); - // The paths above are already validated; a failed write only loses the recording, so - // it must not fail preparation. A later request re-validates and retries the write. - yield* updateMetadataAtomic(fs, path, crypto, metadataPath, { - ...metadata, - requiredRuntimePaths: [...metadata.requiredRuntimePaths, ...appendedPaths], - requiredRuntimeKinds: { ...metadata.requiredRuntimeKinds, ...newKinds }, - }).pipe( - Effect.catch((cause) => - Effect.logWarning("Unable to record newly validated artifact runtime paths", cause), - ), - ); - } - if (request.executablePath !== undefined) { - const executable = safePaths[request.executablePath]; - if (executable === undefined) - return yield* metadataError("Cached artifact executable path is not recorded", { - path: request.executablePath, - }); - yield* ensureExecutableFile(fs, executable.realPath); - } - return Option.some({ - key: request.key, - path: target, - sha256, - requiredRuntimePaths: [...request.requiredRuntimePaths], - ...(request.executablePath === undefined ? {} : { executablePath: request.executablePath }), - outcome: "cached" as const, - }); - }); - const inspectCache = (): Effect.Effect, ArtifactStoreError> => - Effect.gen(function* () { - const exists = yield* mapFs(target, "inspect cached artifact", fs.exists(target)); - if (!exists) return Option.none(); - const cached = yield* checkCached; - if (Option.isSome(cached)) return cached; - const stillExists = yield* mapFs(target, "inspect cached artifact", fs.exists(target)); - if (!stillExists) return Option.none(); - const token = yield* crypto.randomUUIDv4.pipe( - Effect.mapError((cause) => - artifactError(`Unable to allocate artifact replacement name: ${cause.message}`, { - key: request.key, - cause, - }), - ), - ); - const replacement = path.join(targetParent, `.${path.basename(target)}.${token}.invalid`); - const moved = yield* Effect.acquireUseRelease( - fs.rename(target, replacement).pipe( - Effect.as(Option.some(replacement)), - Effect.catch((cause) => - isNotFound(cause) - ? Effect.succeed(Option.none()) - : Effect.fail( - artifactError(`Unable to replace invalid cached artifact: ${String(cause)}`, { - path: target, - cause, - }), - ), - ), - ), - (renamed) => Effect.succeed(Option.isSome(renamed)), - (renamed) => (Option.isSome(renamed) ? cleanup(fs, renamed.value) : Effect.void), - ); - if (moved) { - return Option.none(); - } - const raced = yield* mapFs(target, "inspect cached artifact", fs.exists(target)); - if (!raced) return Option.none(); - const concurrent = yield* checkCached; - if (Option.isSome(concurrent)) return concurrent; - return yield* metadataError("Concurrent cached artifact metadata is invalid", { - path: metadataPath, - }); - }); - const initial = yield* inspectCache(); - if (Option.isSome(initial)) return initial.value; - - // Artifact keys identify immutable published versions. Once a valid entry exists, its - // persisted digest is authoritative; only a cache miss consults the upstream checksum. - const expectedSha256 = yield* source.checksum(request).pipe( - Effect.flatMap((sha256) => - validateSha256(sha256).pipe( - Effect.mapError((cause) => - metadataError("Artifact source returned an invalid SHA-256", { - key: request.key, - cause, - }), - ), + yield* takeLock(connection).pipe( + Effect.mapError((cause) => + artifactError(`Unable to lock artifact staging directory: ${cause.message}`, { + key: request.key, + cause, + }), ), - ), - ); - - const token = yield* crypto.randomUUIDv4.pipe( - Effect.mapError((cause) => - artifactError(`Unable to allocate artifact temporary name: ${cause.message}`, { - key: request.key, - cause, - }), - ), - ); - const staging = path.join(targetParent, `.${path.basename(target)}.${token}.tmp`); - const content = path.join(staging, STAGING_CONTENT_NAME); - const published = yield* Effect.gen(function* () { + ); yield* ensureDirectory(fs, path, content, cacheRoot); const contentRoot = yield* ensureSafeRoot(fs, path, content, cacheRoot); - yield* source.materialize(request, content, expectedSha256, onProgress).pipe( + yield* source.materialize(request, content, resolved.expectedSha256, onProgress).pipe( Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path), Effect.provideService(Crypto.Crypto, crypto), @@ -847,59 +776,107 @@ const makeArtifactOperation = Effect.fn("ArtifactStore.operation")(function* ( // already-owned process service captured by its constructor. Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, childProcessSpawner), ); - const runtimePaths = yield* validateFreshRuntimePaths( + const inspected = yield* validateFreshRuntimePaths( fs, path, content, contentRoot, request.requiredRuntimePaths, ); - const runtimeKinds = Object.fromEntries( - Object.entries(runtimePaths).map(([relative, inspected]) => [relative, inspected.kind]), - ); - yield* writeMetadataSync( - fs, - path.join(content, METADATA_NAME), - metadataFor(request, expectedSha256, runtimeKinds), - ); if (request.executablePath !== undefined) { - const executable = runtimePaths[request.executablePath]; + const executable = inspected[request.executablePath]; if (executable === undefined) return yield* metadataError("Fresh artifact executable path is not recorded", { path: request.executablePath, }); yield* ensureExecutableFile(fs, executable.realPath); - yield* mapFs( - executable.realPath, - "set executable artifact mode", - fs.chmod(executable.realPath, EXECUTABLE_MODE), - ); } - const beforePublish = yield* inspectCache(); - if (Option.isSome(beforePublish)) return beforePublish.value; - const rename = mapFs(content, "publish artifact", fs.rename(content, target)).pipe( - Effect.as(undefined), + yield* mapFs(content, "normalize artifact mode", fs.chmod(content, DIRECTORY_MODE)); + // Protects the narrow window right after publish: the digest lock file's mtime is touched + // here regardless of any pin the caller already holds, so a generation republished after a + // past retirement never inherits its lock file's stale, pre-retirement mtime. + return yield* Effect.scoped( + Pin.pin(resolved.lockPath).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.mapError((cause) => + artifactError(`Unable to pin artifact generation before publishing: ${cause.message}`, { + key: request.key, + cause, + }), + ), + Effect.andThen( + fs.rename(content, resolved.generationPath).pipe( + Effect.as(true as const), + Effect.catch((cause) => + isPublishTargetTaken(cause) + ? Effect.succeed(false as const) + : Effect.fail( + artifactError(`Unable to publish artifact: ${String(cause)}`, { + path: resolved.generationPath, + cause, + }), + ), + ), + Effect.flatMap((won) => + won + ? mapFs( + resolved.keyRoot, + "publish artifact", + fsyncDirectory(fs, resolved.keyRoot), + ) + : Effect.void, + ), + ), + ), + ), ); - const recoverPublish = (): Effect.Effect => - Effect.gen(function* () { - const probe = yield* inspectCache(); - if (Option.isSome(probe)) return probe.value; - return yield* rename; - }); - const recovered: Effect.Effect = - rename.pipe(Effect.catch(recoverPublish)); - return yield* recovered; - }).pipe(Effect.onExit(() => cleanup(fs, staging))); - if (published !== undefined) return published; - return { - key: request.key, - path: target, - sha256: expectedSha256, - requiredRuntimePaths: [...request.requiredRuntimePaths], - ...(request.executablePath === undefined ? {} : { executablePath: request.executablePath }), - outcome: "downloaded" as const, - }; - }); + }), + ).pipe( + Effect.onExit(() => + // Runs after the staging lock connection above has already closed (`Effect.scoped`'s own + // finalizer fires first), so removing its file here is always safe, on every outcome. + cleanup(fs, stagingDir).pipe( + Effect.andThen(fs.remove(stagingLockPath, { force: true })), + Effect.andThen(fs.remove(`${stagingLockPath}-journal`, { force: true })), + Effect.ignore, + ), + ), + ); + const hit = yield* checkHit(fs, path, cacheRoot, request, resolved); + if (Option.isNone(hit)) + return yield* artifactError("Unable to publish artifact: the published generation vanished", { + path: resolved.generationPath, + }); + return { ...hit.value, outcome: "downloaded" as const }; +}); + +const prepareOrResolve = Effect.fn("ArtifactStore.prepareOrResolve")(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + crypto: Crypto.Crypto, + childProcessSpawner: ChildProcessSpawner.ChildProcessSpawner["Service"], + cacheRoot: string, + source: ArtifactSource, + request: ArtifactRequest, + resolved: ResolvedGeneration, + onProgress?: (state: "downloading" | "preparing") => void, +): Effect.fn.Return { + yield* ensureDirectory(fs, path, resolved.keyRoot, cacheRoot); + yield* reapStaleStaging(fs, path, resolved.stagingRoot); + yield* retireStaleGenerations(fs, path, crypto, cacheRoot); + const hit = yield* checkHit(fs, path, cacheRoot, request, resolved); + if (Option.isSome(hit)) return hit.value; + return yield* publishGeneration( + fs, + path, + crypto, + childProcessSpawner, + cacheRoot, + source, + request, + resolved, + onProgress, + ); }); export const makeArtifactStore = Effect.fn("ArtifactStore.makeStore")(function* ( @@ -936,15 +913,72 @@ export const makeArtifactStore = Effect.fn("ArtifactStore.makeStore")(function* if (rootInfo.type !== "Directory") return yield* artifactError("Artifact cache root must be a directory", { path: cacheRoot }); yield* mapFs(cacheRoot, "secure artifact cache root", restrictDirectoryToOwner(fs, cacheRoot)); + const prepare = Effect.fn("ArtifactStore.prepare")(function* ( request: ArtifactRequest, onProgress?: (state: "downloading" | "preparing") => void, ) { yield* validateRequest(request); - const target = path.resolve(cacheRoot, request.key); - if (!pathWithin(cacheRoot, target, path.sep)) - return yield* artifactError("Artifact key escapes cache root", { key: request.key }); - const prepared = yield* makeArtifactOperation( + const resolved = yield* resolveGeneration(path, cacheRoot, options.source, request); + // Ahead-of-time preparation pins nothing for its caller, but a warm generation's own + // inspection and a fresh one's publication both still need protection from a concurrent + // retirement sweep elsewhere; a SHARED pin held only for this call's own duration, released + // before returning, gives that without leaving anything pinned for the caller. + yield* ensureDirectory(fs, path, resolved.keyRoot, cacheRoot); + const prepared = yield* Effect.scoped( + Pin.pin(resolved.lockPath).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.mapError((cause) => + artifactError(`Unable to pin artifact generation: ${cause.message}`, { + key: request.key, + cause, + }), + ), + Effect.andThen( + prepareOrResolve( + fs, + path, + crypto, + childProcessSpawner, + cacheRoot, + options.source, + request, + resolved, + onProgress, + ), + ), + ), + ); + yield* Effect.annotateCurrentSpan({ + "artifact.outcome": prepared.outcome, + "artifact.digest": resolved.expectedSha256, + }); + return prepared; + }); + + /** + * The one scoped launch-time operation: pins the generation, resolves or prepares it, and + * returns its paths. Every consumer must use those paths only inside this scope. + */ + const use = Effect.fn("ArtifactStore.use")(function* ( + request: ArtifactRequest, + onProgress?: (state: "downloading" | "preparing") => void, + ) { + yield* validateRequest(request); + const resolved = yield* resolveGeneration(path, cacheRoot, options.source, request); + // The digest lock file's parent must exist before it can be created; `prepareOrResolve` below + // ensures it again, idempotently, before resolving or preparing the generation itself. + yield* ensureDirectory(fs, path, resolved.keyRoot, cacheRoot); + yield* Pin.pin(resolved.lockPath).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.mapError((cause) => + artifactError(`Unable to pin artifact generation: ${cause.message}`, { + key: request.key, + cause, + }), + ), + ); + const prepared = yield* prepareOrResolve( fs, path, crypto, @@ -952,10 +986,15 @@ export const makeArtifactStore = Effect.fn("ArtifactStore.makeStore")(function* cacheRoot, options.source, request, + resolved, onProgress, ); - yield* Effect.annotateCurrentSpan({ "artifact.outcome": prepared.outcome }); + yield* Effect.annotateCurrentSpan({ + "artifact.outcome": prepared.outcome, + "artifact.digest": resolved.expectedSha256, + }); return prepared; }); - return { prepare }; + + return { prepare, use }; }); diff --git a/packages/stack/src/preparation/artifacts.integration.test.ts b/packages/stack/src/preparation/artifacts.integration.test.ts index 8f637d0a42..cecb3699d6 100644 --- a/packages/stack/src/preparation/artifacts.integration.test.ts +++ b/packages/stack/src/preparation/artifacts.integration.test.ts @@ -3,17 +3,27 @@ import { describe, expect, it } from "@effect/vitest"; import { Cause, Crypto, + Data, DateTime, Deferred, + Duration, Effect, Exit, Fiber, FileSystem, Layer, Option, - PlatformError, - Schema, + Path, + Schedule, + Scope, + Stream, } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { createHash } from "node:crypto"; +import { fileURLToPath } from "node:url"; +import * as Environment from "../namespace/Environment.ts"; +import { defaultNativeProcessLauncher, spawnNativeProcess } from "../runtime/NativeProcess.ts"; +import { acquireLock, isBusy, takeExclusiveLock } from "../namespace/drivers/Sqlite.ts"; import { ArtifactIntegrityError, PreparationError } from "./Errors.ts"; import { makeArtifactStore, type ArtifactRequest, type ArtifactSource } from "./ArtifactStore.ts"; import { verifySha256 } from "./Integrity.ts"; @@ -33,6 +43,18 @@ const errorOf = (exit: Exit.Exit): E | undefined => const withPlatform = (effect: Effect.Effect) => Effect.scoped(effect).pipe(Effect.provide(layer)); +const mapMaterializeError = (effect: Effect.Effect) => + effect.pipe( + Effect.mapError((cause) => + cause instanceof PreparationError || cause instanceof ArtifactIntegrityError + ? cause + : new PreparationError({ + message: `materialization failed: ${cause instanceof Error ? cause.message : String(cause)}`, + cause, + }), + ), + ); + const sourceWriting = (bytes: Uint8Array = archive): ArtifactSource => ({ checksum: () => Effect.succeed(archiveSha256), materialize: (_request, destination, expectedSha256) => @@ -40,45 +62,96 @@ const sourceWriting = (bytes: Uint8Array = archive): ArtifactSource => ({ const fs = yield* FileSystem.FileSystem; yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); yield* fs.makeDirectory(`${destination}/etc`, { recursive: true }); - yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres"); - yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config"); + yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres", { mode: 0o755 }); + yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config", { mode: 0o644 }); const crypto = yield* Crypto.Crypto; yield* verifySha256(bytes, expectedSha256).pipe(Effect.provideService(Crypto.Crypto, crypto)); - }).pipe( - Effect.mapError((cause) => - cause instanceof ArtifactIntegrityError - ? cause - : new PreparationError({ - message: `materialization failed: ${cause instanceof Error ? cause.message : String(cause)}`, - cause, - }), - ), - ), + }).pipe(mapMaterializeError), }); -const replacesMalformedMetadata = (label: string, metadata: string | undefined) => - withPlatform( +/** A second, sibling digest under the same key: used to force a retirement sweep as a side effect + * of an unrelated `prepare`, without redownloading or recreating the generation under test. */ +const archiveAlt = new TextEncoder().encode("archive-alt"); +const archiveAltSha256 = createHash("sha256").update(archiveAlt).digest("hex"); +const sourceWritingAlt = (): ArtifactSource => ({ + checksum: () => Effect.succeed(archiveAltSha256), + materialize: (_request, destination, expectedSha256) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: `supabase-stack-artifact-${label}-metadata-`, + yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); + yield* fs.makeDirectory(`${destination}/etc`, { recursive: true }); + yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres alt", { + mode: 0o755, }); - const target = `${root}/${request.key}`; - yield* fs.makeDirectory(`${target}/bin`, { recursive: true }); - yield* fs.writeFileString(`${target}/bin/postgres`, "unverified postgres"); - if (metadata !== undefined) yield* fs.writeFileString(`${target}/.artifact.json`, metadata); - - const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); - const prepared = yield* store.prepare(request); - - expect(prepared.outcome).toBe("downloaded"); - expect(yield* fs.readFileString(`${target}/bin/postgres`)).toBe("native postgres"); - expect(yield* fs.exists(`${target}/.artifact.json`)).toBe(true); - }), + yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config alt", { mode: 0o644 }); + const crypto = yield* Crypto.Crypto; + yield* verifySha256(archiveAlt, expectedSha256).pipe( + Effect.provideService(Crypto.Crypto, crypto), + ); + }).pipe(mapMaterializeError), +}); + +/** Sets `path`'s mtime 31 days in the past: a legitimate test-only lever on retirement's age fence. */ +const ageLockFile = (fs: FileSystem.FileSystem, path: string) => + Effect.gen(function* () { + const now = yield* DateTime.now; + const aged = DateTime.toDate(DateTime.subtract(now, { days: 31 })); + yield* fs.utimes(path, aged, aged); + }); + +/** Matches a staging token's own lock file, or the rollback journal SQLite leaves transiently. */ +const isStagingLockFile = (name: string) => /\.lock(-journal)?$/u.test(name); + +const digestDirectories = (fs: FileSystem.FileSystem, keyRoot: string) => + fs.readDirectory(keyRoot).pipe( + Effect.orElseSucceed(() => []), + Effect.map((names) => names.filter((name) => /^[0-9a-f]{64}$/u.test(name))), + ); + +const fixturePath = fileURLToPath( + new URL("../../tests/artifact-store-fixture.ts", import.meta.url), +); + +const spawnFixture = (args: ReadonlyArray) => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* spawner.spawn( + ChildProcess.make(process.execPath, [fixturePath, ...args], { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + detached: true, + forceKillAfter: "1 second", + }), + ); + }); + +const runToCompletion = (handle: ChildProcessSpawner.ChildProcessHandle) => + Effect.all( + [ + handle.stdout.pipe(Stream.decodeText, Stream.mkString), + handle.stderr.pipe(Stream.decodeText, Stream.mkString), + handle.exitCode, + ], + { concurrency: "unbounded" }, + ); + +/** Reads the fixture's first stdout line without waiting for it to exit: `use`/`stall` block forever. */ +const firstLine = (handle: ChildProcessSpawner.ChildProcessHandle) => + handle.stdout.pipe( + Stream.decodeText, + Stream.splitLines, + Stream.runHead, + Effect.map(Option.getOrThrow), + ); + +const killAndAwaitExit = (handle: ChildProcessSpawner.ChildProcessHandle) => + Effect.sync(() => process.kill(Number(handle.pid), "SIGKILL")).pipe( + Effect.andThen(handle.exitCode.pipe(Effect.ignore)), ); describe("verified native artifact preparation", () => { - it.live("downloads and atomically publishes an executable artifact tree", () => + it.live("downloads and publishes a content-addressed generation", () => withPlatform( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -86,8 +159,12 @@ describe("verified native artifact preparation", () => { const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); const prepared = yield* store.prepare(request); expect(prepared.outcome).toBe("downloaded"); + expect(prepared.path).toBe(`${yield* fs.realPath(root)}/${request.key}/${archiveSha256}`); expect(yield* fs.exists(`${prepared.path}/bin/postgres`)).toBe(true); - expect((yield* fs.stat(`${prepared.path}/bin/postgres`)).mode & 0o111).not.toBe(0); + expect((yield* fs.stat(prepared.path)).mode & 0o777).toBe(0o755); + expect((yield* fs.stat(`${prepared.path}/bin/postgres`)).mode & 0o777).toBe(0o755); + expect((yield* fs.stat(`${prepared.path}/etc/postgres.conf`)).mode & 0o777).toBe(0o644); + expect(yield* fs.exists(prepared.lockPath)).toBe(true); }), ), ); @@ -142,14 +219,15 @@ describe("verified native artifact preparation", () => { const second = yield* store.prepare(request); expect(first.outcome).toBe("downloaded"); expect(second.outcome).toBe("cached"); - expect(checksumCalls).toBe(1); + expect(second.path).toBe(first.path); + expect(checksumCalls).toBe(2); expect(materializeCalls).toBe(1); }), ), ); it.live( - "keeps the published tree and reuses it when required runtime paths expand on the same key", + "serves a newly required path already present in the published tree without redownloading", () => withPlatform( Effect.gen(function* () { @@ -157,14 +235,9 @@ describe("verified native artifact preparation", () => { const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-artifact-paths-expand-", }); - let checksumCalls = 0; let materializeCalls = 0; const source: ArtifactSource = { - checksum: () => - Effect.sync(() => { - checksumCalls += 1; - return archiveSha256; - }), + checksum: () => Effect.succeed(archiveSha256), materialize: (entry, destination) => Effect.sync(() => { materializeCalls += 1; @@ -184,232 +257,12 @@ describe("verified native artifact preparation", () => { expect(second.path).toBe(first.path); expect(stillPublishedIno).toEqual(publishedIno); expect(second.requiredRuntimePaths).toEqual([...request.requiredRuntimePaths]); - expect(checksumCalls).toBe(1); expect(materializeCalls).toBe(1); - expect(yield* fs.readFileString(`${second.path}/bin/postgres`)).toBe("native postgres"); - const metadata = yield* Schema.decodeEffect( - Schema.fromJsonString( - Schema.Struct({ - requiredRuntimePaths: Schema.Array(Schema.String), - requiredRuntimeKinds: Schema.Record(Schema.String, Schema.String), - }), - ), - )(yield* fs.readFileString(`${second.path}/.artifact.json`)); - expect(metadata.requiredRuntimePaths).toContain("etc/postgres.conf"); - expect(metadata.requiredRuntimeKinds["etc/postgres.conf"]).toBe("file"); + expect(yield* fs.readFileString(`${second.path}/etc/postgres.conf`)).toBe("config"); }), ), ); - it.live("replaces a cached tree with unknown artifact metadata", () => - replacesMalformedMetadata("unknown", '{"format":"supabase-stack-artifact-v0"}'), - ); - - it.live("replaces a cached tree with missing artifact metadata", () => - replacesMalformedMetadata("missing", undefined), - ); - - it.live("reuses a cache hit after required file contents are modified", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-tampered-runtime-", - }); - const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); - const published = yield* store.prepare(request); - yield* fs.writeFileString(`${published.path}/bin/postgres`, "tampered executable"); - const cached = yield* store.prepare(request); - expect(cached.outcome).toBe("cached"); - }), - ), - ); - - it.live("reuses cached directories without hashing their contents", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-directory-runtime-", - }); - const directoryRequest: ArtifactRequest = { - ...request, - key: "database/postgres-directory-runtime", - requiredRuntimePaths: ["bin/postgres", "share/runtime"], - }; - const source: ArtifactSource = { - checksum: () => Effect.succeed(archiveSha256), - materialize: (_entry, destination) => - Effect.gen(function* () { - yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); - yield* fs.makeDirectory(`${destination}/share/runtime`, { recursive: true }); - yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres"); - yield* fs.writeFileString(`${destination}/share/runtime/config`, "config"); - yield* fs.symlink("config", `${destination}/share/runtime/config-link`); - return; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), - }; - const store = yield* makeArtifactStore({ cacheRoot: root, source }); - const prepared = yield* store.prepare(directoryRequest); - expect((yield* store.prepare(directoryRequest)).outcome).toBe("cached"); - - yield* fs.writeFileString(`${prepared.path}/share/runtime/config`, "tampered config"); - expect((yield* store.prepare(directoryRequest)).outcome).toBe("cached"); - }), - ), - ); - - it.live("rejects a non-EINVAL readLink failure on a cache hit", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-cache-readlink-error-", - }); - const publisher = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); - yield* publisher.prepare(request); - const failingFs: FileSystem.FileSystem = { - ...fs, - readLink: (candidate) => - candidate.endsWith("/bin/postgres") - ? Effect.fail( - PlatformError.systemError({ - _tag: "PermissionDenied", - module: "FileSystem", - method: "readLink", - pathOrDescriptor: candidate, - cause: { code: "EACCES" }, - }), - ) - : fs.readLink(candidate), - }; - const checker = yield* makeArtifactStore({ - cacheRoot: root, - source: sourceWriting(), - }).pipe(Effect.provideService(FileSystem.FileSystem, failingFs)); - - const exit = yield* checker.prepare(request).pipe(Effect.exit); - const error = errorOf(exit); - - expect(error).toBeInstanceOf(ArtifactIntegrityError); - expect(error?.message).toContain("Unable to inspect required runtime path"); - }), - ), - ); - - it.live("does not recursively enumerate required directories on a cache hit", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-cache-no-recursion-", - }); - const directoryRequest: ArtifactRequest = { - ...request, - key: "database/postgres-cache-no-recursion", - requiredRuntimePaths: ["bin/postgres", "share/runtime"], - }; - const source: ArtifactSource = { - checksum: () => Effect.succeed(archiveSha256), - materialize: (_entry, destination) => - Effect.gen(function* () { - yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); - yield* fs.makeDirectory(`${destination}/share/runtime`, { recursive: true }); - yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres"); - yield* fs.writeFileString(`${destination}/share/runtime/config`, "config"); - return; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), - }; - const publisher = yield* makeArtifactStore({ cacheRoot: root, source }); - yield* publisher.prepare(directoryRequest); - const guardedFs: FileSystem.FileSystem = { - ...fs, - readDirectory: (candidate, options) => - candidate.endsWith("/share/runtime") - ? Effect.fail( - PlatformError.systemError({ - _tag: "Busy", - module: "FileSystem", - method: "readDirectory", - pathOrDescriptor: candidate, - cause: { code: "EBUSY" }, - }), - ) - : fs.readDirectory(candidate, options), - }; - const checker = yield* makeArtifactStore({ - cacheRoot: root, - source, - }).pipe(Effect.provideService(FileSystem.FileSystem, guardedFs)); - - const cached = yield* checker.prepare(directoryRequest); - - expect(cached.outcome).toBe("cached"); - }), - ), - ); - - it.live("rejects a missing required path on a cache hit", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-cache-missing-", - }); - const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); - const published = yield* store.prepare(request); - yield* fs.remove(`${published.path}/etc/postgres.conf`); - - const exit = yield* store.prepare(request).pipe(Effect.exit); - - expect(errorOf(exit)).toBeInstanceOf(ArtifactIntegrityError); - }), - ), - ); - - it.live("rejects a cache hit whose newly required path is missing from the published tree", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-cache-new-path-missing-", - }); - const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); - const narrow: ArtifactRequest = { ...request, requiredRuntimePaths: ["bin/postgres"] }; - const published = yield* store.prepare(narrow); - const publishedIno = (yield* fs.stat(published.path)).ino; - - const wider: ArtifactRequest = { - ...request, - requiredRuntimePaths: ["bin/postgres", "share/missing"], - }; - const exit = yield* store.prepare(wider).pipe(Effect.exit); - - const error = errorOf(exit); - expect(error).toBeInstanceOf(ArtifactIntegrityError); - expect(error?.message).toContain(`remove ${published.path} to download it again`); - expect(yield* fs.exists(published.path)).toBe(true); - expect((yield* fs.stat(published.path)).ino).toEqual(publishedIno); - }), - ), - ); - it.live("rejects a required path whose basic kind changes on a cache hit", () => withPlatform( Effect.gen(function* () { @@ -441,31 +294,8 @@ describe("verified native artifact preparation", () => { }); const outsideExecutable = `${outside}/postgres`; yield* fs.writeFileString(outsideExecutable, "outside"); - const symlinkRequest: ArtifactRequest = { - ...request, - key: "database/postgres-cache-link", - }; - const source: ArtifactSource = { - checksum: () => Effect.succeed(archiveSha256), - materialize: (_entry, destination) => - Effect.gen(function* () { - yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); - yield* fs.makeDirectory(`${destination}/etc`, { recursive: true }); - yield* fs.writeFileString(`${destination}/bin/postgres.real`, "native postgres"); - yield* fs.symlink("postgres.real", `${destination}/bin/postgres`); - yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config"); - return; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), - }; - const store = yield* makeArtifactStore({ cacheRoot: root, source }); + const symlinkRequest: ArtifactRequest = { ...request, key: "database/postgres-cache-link" }; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); const published = yield* store.prepare(symlinkRequest); yield* fs.remove(`${published.path}/bin/postgres`); yield* fs.symlink(outsideExecutable, `${published.path}/bin/postgres`); @@ -477,7 +307,7 @@ describe("verified native artifact preparation", () => { ), ); - it.live("rejects a checksum mismatch and removes only its temporary tree", () => + it.live("rejects a checksum mismatch and leaves no generation or staging behind", () => withPlatform( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -490,9 +320,11 @@ describe("verified native artifact preparation", () => { }); const exit = yield* store.prepare(request).pipe(Effect.exit); expect(errorOf(exit)).toBeInstanceOf(ArtifactIntegrityError); - const entries = yield* fs.readDirectory(root, { recursive: true }); - expect(entries.some((entry) => entry.endsWith(".tmp"))).toBe(false); - expect(yield* fs.exists(`${root}/database/postgres`)).toBe(false); + expect(yield* digestDirectories(fs, `${root}/${request.key}`)).toEqual([]); + const staging = yield* fs + .readDirectory(`${root}/${request.key}/.staging`) + .pipe(Effect.orElseSucceed(() => [])); + expect(staging).toEqual([]); }), ), ); @@ -508,27 +340,17 @@ describe("verified native artifact preparation", () => { checksum: () => Effect.succeed(archiveSha256), materialize: (_entry, destination) => Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; yield* fs.makeDirectory(`${destination}/bin/postgres`, { recursive: true }); yield* fs.makeDirectory(`${destination}/etc`, { recursive: true }); yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config"); - return; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), + }).pipe(mapMaterializeError), }; const store = yield* makeArtifactStore({ cacheRoot: root, source }); const exit = yield* store.prepare(request).pipe(Effect.exit); expect(errorOf(exit)).toBeInstanceOf(ArtifactIntegrityError); - const entries = yield* fs.readDirectory(root, { recursive: true }); - expect(entries.some((entry) => entry.endsWith(".tmp"))).toBe(false); - expect(yield* fs.exists(`${root}/database/postgres`)).toBe(false); + expect(yield* digestDirectories(fs, `${root}/${request.key}`)).toEqual([]); }), ), ); @@ -544,21 +366,17 @@ describe("verified native artifact preparation", () => { checksum: () => Effect.succeed(archiveSha256), materialize: (_entry, destination) => Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); yield* fs.makeDirectory(`${destination}/etc`, { recursive: true }); - yield* fs.writeFileString(`${destination}/bin/postgres.real`, "native postgres"); + yield* fs.writeFileString(`${destination}/bin/postgres.real`, "native postgres", { + mode: 0o755, + }); yield* fs.symlink("postgres.real", `${destination}/bin/postgres`); - yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config"); - return; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), + yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config", { + mode: 0o644, + }); + }).pipe(mapMaterializeError), }; const store = yield* makeArtifactStore({ cacheRoot: root, source }); const prepared = yield* store.prepare(request); @@ -586,123 +404,19 @@ describe("verified native artifact preparation", () => { checksum: () => Effect.succeed(archiveSha256), materialize: (_entry, destination) => Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); yield* fs.makeDirectory(`${destination}/etc`, { recursive: true }); yield* fs.symlink(outsideExecutable, `${destination}/bin/postgres`); yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config"); - return; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), + }).pipe(mapMaterializeError), }; const store = yield* makeArtifactStore({ cacheRoot: root, source }); const exit = yield* store.prepare(request).pipe(Effect.exit); expect(errorOf(exit)).toBeInstanceOf(ArtifactIntegrityError); expect(yield* fs.exists(outsideExecutable)).toBe(true); - expect(yield* fs.exists(`${root}/database/postgres`)).toBe(false); - }), - ), - ); - - it.live("rejects an escaping symlink nested in a fresh required directory", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-nested-link-escape-", - }); - const outside = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-nested-link-outside-", - }); - const outsideConfig = `${outside}/config`; - yield* fs.writeFileString(outsideConfig, "outside"); - const directoryRequest: ArtifactRequest = { - ...request, - key: "database/postgres-nested-link", - requiredRuntimePaths: ["bin/postgres", "share/runtime"], - }; - const source: ArtifactSource = { - checksum: () => Effect.succeed(archiveSha256), - materialize: (_entry, destination) => - Effect.gen(function* () { - yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); - yield* fs.makeDirectory(`${destination}/share/runtime`, { recursive: true }); - yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres"); - yield* fs.symlink(outsideConfig, `${destination}/share/runtime/config`); - return; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), - }; - const store = yield* makeArtifactStore({ cacheRoot: root, source }); - const exit = yield* store.prepare(directoryRequest).pipe(Effect.exit); - - expect(errorOf(exit)).toBeInstanceOf(ArtifactIntegrityError); - expect(yield* fs.exists(`${root}/database/postgres-nested-link`)).toBe(false); - }), - ), - ); - - it.live("rejects an escaping symlink nested in a newly required directory on a cache hit", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-cache-new-nested-link-escape-", - }); - const outside = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-cache-new-nested-link-outside-", - }); - const outsideConfig = `${outside}/config`; - yield* fs.writeFileString(outsideConfig, "outside"); - const directoryRequest: ArtifactRequest = { - ...request, - key: "database/postgres-cache-new-nested-link", - requiredRuntimePaths: ["bin/postgres", "share/runtime"], - }; - const source: ArtifactSource = { - checksum: () => Effect.succeed(archiveSha256), - materialize: (_entry, destination) => - Effect.gen(function* () { - yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); - yield* fs.makeDirectory(`${destination}/share/runtime`, { recursive: true }); - yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres"); - yield* fs.symlink(outsideConfig, `${destination}/share/runtime/config`); - return; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), - }; - const store = yield* makeArtifactStore({ cacheRoot: root, source }); - const narrow: ArtifactRequest = { - ...directoryRequest, - requiredRuntimePaths: ["bin/postgres"], - }; - const published = yield* store.prepare(narrow); - - const exit = yield* store.prepare(directoryRequest).pipe(Effect.exit); - - expect(errorOf(exit)).toBeInstanceOf(ArtifactIntegrityError); - expect(yield* fs.exists(published.path)).toBe(true); + expect(yield* digestDirectories(fs, `${root}/${request.key}`)).toEqual([]); }), ), ); @@ -719,31 +433,22 @@ describe("verified native artifact preparation", () => { checksum: () => Effect.succeed(archiveSha256), materialize: (_entry, destination) => Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; yield* fs.writeFileString(`${destination}/partial`, "partial"); yield* Deferred.succeed(started, undefined); return yield* Effect.never; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), + }).pipe(mapMaterializeError), }; const store = yield* makeArtifactStore({ cacheRoot: root, source }); const preparation = yield* Effect.forkChild(store.prepare(request)); yield* Deferred.await(started); yield* Fiber.interrupt(preparation); - const entries = yield* fs.readDirectory(root, { recursive: true }); - expect(entries.some((entry) => entry.endsWith(".tmp"))).toBe(false); - expect(yield* fs.exists(`${root}/database/postgres`)).toBe(false); + expect(yield* digestDirectories(fs, `${root}/${request.key}`)).toEqual([]); }), ), ); - it.live("allows separate stores to duplicate work and converge safely", () => + it.live("allows separate stores to duplicate work and converge on one published generation", () => withPlatform( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -756,27 +461,23 @@ describe("verified native artifact preparation", () => { let calls = 0; const source: ArtifactSource = { checksum: () => Effect.succeed(archiveSha256), - materialize: (entry, destination) => + materialize: (_entry, destination) => Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; calls += 1; yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); yield* fs.makeDirectory(`${destination}/etc`, { recursive: true }); - yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres"); - yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config"); + yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres", { + mode: 0o755, + }); + yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config", { + mode: 0o644, + }); yield* calls === 1 ? Deferred.succeed(firstStarted, undefined) : Deferred.succeed(secondStarted, undefined); yield* Deferred.await(release); - return; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), + }).pipe(mapMaterializeError), }; const firstStore = yield* makeArtifactStore({ cacheRoot: root, source }); const secondStore = yield* makeArtifactStore({ cacheRoot: root, source }); @@ -789,6 +490,10 @@ describe("verified native artifact preparation", () => { expect(calls).toBe(2); expect(results[0].path).toBe(results[1].path); expect(yield* fs.exists(`${results[0].path}/bin/postgres`)).toBe(true); + const staging = yield* fs + .readDirectory(`${root}/${request.key}/.staging`) + .pipe(Effect.orElseSucceed(() => [])); + expect(staging).toEqual([]); }), ), ); @@ -796,16 +501,13 @@ describe("verified native artifact preparation", () => { it.live("rejects traversal in keys and required runtime paths before touching the source", () => withPlatform( Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-artifact-path-" }); + const root = yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-path-", + }); let called = false; const source: ArtifactSource = { checksum: () => Effect.succeed(archiveSha256), - materialize: () => - Effect.sync(() => { - called = true; - return; - }), + materialize: () => Effect.sync(() => (called = true)), }; const store = yield* makeArtifactStore({ cacheRoot: root, source }); const keyExit = yield* store.prepare({ ...request, key: "../escape" }).pipe(Effect.exit); @@ -831,11 +533,7 @@ describe("verified native artifact preparation", () => { let called = false; const source: ArtifactSource = { checksum: () => Effect.succeed(archiveSha256), - materialize: () => - Effect.sync(() => { - called = true; - return; - }), + materialize: () => Effect.sync(() => (called = true)), }; const store = yield* makeArtifactStore({ cacheRoot: root, source }); const exit = yield* store.prepare({ ...request, key: "nested/postgres" }).pipe(Effect.exit); @@ -848,110 +546,492 @@ describe("verified native artifact preparation", () => { ); }); -describe("orphaned temp/quarantine sweep", () => { - it.live("removes a prepared key's old leftovers and keeps fresh and unrelated ones", () => +describe("a required path missing from the archive", () => { + const narrowSource = (): ArtifactSource => ({ + checksum: () => Effect.succeed(archiveSha256), + materialize: (_request, destination, expectedSha256) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); + yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres", { + mode: 0o755, + }); + const crypto = yield* Crypto.Crypto; + yield* verifySha256(archive, expectedSha256).pipe( + Effect.provideService(Crypto.Crypto, crypto), + ); + }).pipe(mapMaterializeError), + }); + + it.live("fails with a typed error on a cold cache, and leaves the cache unchanged", () => withPlatform( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-sweep-", + prefix: "supabase-stack-artifact-missing-cold-", }); - const now = yield* DateTime.now; - const old = DateTime.toDate(DateTime.subtract(now, { hours: 25 })); - const recent = DateTime.toDate(DateTime.subtract(now, { hours: 1 })); - const parent = `${root}/database`; - const oldTemp = `${parent}/.postgres.00000000-0000-4000-8000-000000000001.tmp`; - const oldQuarantine = `${parent}/.postgres.00000000-0000-4000-8000-000000000002.invalid`; - const freshTemp = `${parent}/.postgres.00000000-0000-4000-8000-000000000003.tmp`; - const freshQuarantine = `${parent}/.postgres.00000000-0000-4000-8000-000000000004.invalid`; - const unrelated = `${parent}/nested/.x.tmp`; - const leftovers = [oldTemp, oldQuarantine, freshTemp, freshQuarantine, unrelated]; - for (const leftover of leftovers) { - yield* fs.makeDirectory(leftover, { recursive: true }); - yield* fs.writeFileString(`${leftover}/marker`, "leftover"); - } - for (const leftover of [oldTemp, oldQuarantine, unrelated]) - yield* fs.utimes(leftover, old, old); - for (const leftover of [freshTemp, freshQuarantine]) - yield* fs.utimes(leftover, recent, recent); - - const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); - expect(yield* fs.exists(oldTemp), "construction leaves the cache alone").toBe(true); - expect((yield* store.prepare(request)).outcome).toBe("downloaded"); - - expect(yield* fs.exists(oldTemp)).toBe(false); - expect(yield* fs.exists(oldQuarantine)).toBe(false); - expect(yield* fs.exists(freshTemp)).toBe(true); - expect(yield* fs.exists(freshQuarantine)).toBe(true); - expect(yield* fs.exists(unrelated)).toBe(true); + const key = "database/postgres-missing-cold"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: narrowSource() }); + const wideRequest: ArtifactRequest = { ...request, key }; + const exit = yield* store.prepare(wideRequest).pipe(Effect.exit); + expect(errorOf(exit)).toBeInstanceOf(ArtifactIntegrityError); + expect(yield* digestDirectories(fs, `${root}/${key}`)).toEqual([]); }), ), ); - it.live("never reaps an in-progress extraction whose source backdated its own destination", () => + it.live("fails with a typed error on a warm cache, and leaves the cache unchanged", () => withPlatform( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const crypto = yield* Crypto.Crypto; const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-reaper-race-", + prefix: "supabase-stack-artifact-missing-warm-", }); - const backdated = yield* Deferred.make(); - const secondReachedMaterialize = yield* Deferred.make(); - let materializeCalls = 0; - const source: ArtifactSource = { - checksum: () => Effect.succeed(archiveSha256), - materialize: (_entry, destination, expectedSha256) => - Effect.gen(function* () { - materializeCalls += 1; - if (materializeCalls > 1) { - // The second preparation arrives here after running the reaper; it fails instead - // of racing the first to publish, which keeps the interleaving deterministic. - yield* Deferred.succeed(secondReachedMaterialize, undefined); - return yield* new PreparationError({ - message: "the second preparation stops before publishing", - }); - } - yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); - yield* fs.makeDirectory(`${destination}/etc`, { recursive: true }); - yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres"); - yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config"); - yield* verifySha256(archive, expectedSha256).pipe( - Effect.provideService(Crypto.Crypto, crypto), - ); - // GNU tar sets an extraction destination's own mtime from the archive's `./` - // entry; the slim-services archives carry epoch mtimes there. - yield* fs.utimes(destination, 0, 0); - yield* Deferred.succeed(backdated, undefined); - yield* Deferred.await(secondReachedMaterialize); - }).pipe( - Effect.mapError((cause) => - cause instanceof PreparationError || cause instanceof ArtifactIntegrityError - ? cause - : new PreparationError({ - message: `materialization failed: ${cause instanceof Error ? cause.message : String(cause)}`, - cause, - }), - ), - ), + const key = "database/postgres-missing-warm"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: narrowSource() }); + const narrowRequest: ArtifactRequest = { + ...request, + key, + requiredRuntimePaths: ["bin/postgres"], }; - const store = yield* makeArtifactStore({ cacheRoot: root, source }); + const published = yield* store.prepare(narrowRequest); + expect(published.outcome).toBe("downloaded"); + + const wideRequest: ArtifactRequest = { ...request, key }; + const exit = yield* store.prepare(wideRequest).pipe(Effect.exit); + expect(errorOf(exit)).toBeInstanceOf(ArtifactIntegrityError); + expect(yield* digestDirectories(fs, `${root}/${key}`)).toEqual([archiveSha256]); + expect(yield* fs.exists(published.path)).toBe(true); + }), + ), + ); +}); - const first = yield* Effect.forkChild(store.prepare(request)); - yield* Deferred.await(backdated); +describe("concurrent preparers across processes", () => { + it.live( + "both succeed, exactly one generation is published, and no staging is left", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-cross-process-", + }); + const key = "database/postgres-cross-process"; + const handles = yield* Effect.all( + [spawnFixture(["prepare", root, key]), spawnFixture(["prepare", root, key])], + { concurrency: "unbounded" }, + ); + const results = yield* Effect.all(handles.map(runToCompletion), { + concurrency: "unbounded", + }); + const parsed = results.map(([stdout, stderr, exitCode]) => { + expect(Number(exitCode), stderr).toBe(0); + const match = /prepared:(\w+):(.+):([0-9a-f]+)/.exec(stdout.trim()); + if (match === null) throw new Error(`Unparseable fixture output: ${stdout}`); + return { outcome: match[1], path: match[2], sha256: match[3] }; + }); + expect(parsed[0]?.path).toBe(parsed[1]?.path); + expect(parsed[0]?.sha256).toBe(parsed[1]?.sha256); + const staging = yield* fs + .readDirectory(`${root}/${key}/.staging`) + .pipe(Effect.orElseSucceed(() => [])); + expect(staging).toEqual([]); + }), + ), + 20_000, + ); +}); - // Runs the same leftover reaper the first operation's staging directory is exposed to; - // it refuses its own materialization once it gets there, so it can never win a race to - // publish and mask a reap that already happened. - const second = yield* store.prepare(request).pipe(Effect.exit); - expect(Exit.isFailure(second)).toBe(true); +describe("lock-guarded staging", () => { + it.live( + "reaps a SIGKILLed preparer's staging directory on the next preparer, which publishes normally", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-sigkill-reap-", + }); + const key = "database/postgres-sigkill-reap"; + const handle = yield* spawnFixture(["stall", root, key]); + yield* firstLine(handle); + const stagingRoot = `${root}/${key}/.staging`; + const tokensBefore = (yield* fs.readDirectory(stagingRoot)).filter( + (name) => !isStagingLockFile(name), + ); + expect(tokensBefore).toHaveLength(1); - const published = yield* Fiber.join(first); - expect(published.outcome).toBe("downloaded"); - expect(yield* fs.exists(`${published.path}/bin/postgres`)).toBe(true); - expect(yield* fs.readFileString(`${published.path}/bin/postgres`)).toBe("native postgres"); - expect(yield* fs.readFileString(`${published.path}/etc/postgres.conf`)).toBe("config"); + yield* killAndAwaitExit(handle); + + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const prepared = yield* store.prepare({ ...request, key }); + expect(prepared.outcome).toBe("downloaded"); + const tokensAfter = (yield* fs.readDirectory(stagingRoot)).filter( + (name) => !isStagingLockFile(name), + ); + expect(tokensAfter).toHaveLength(0); + }), + ), + 20_000, + ); + + it.live( + "never touches a live preparer's staging directory while another preparer runs", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-live-stage-", + }); + const key = "database/postgres-live-stage"; + const handle = yield* spawnFixture(["stall", root, key]); + yield* firstLine(handle); + const stagingRoot = `${root}/${key}/.staging`; + const tokensBefore = (yield* fs.readDirectory(stagingRoot)).filter( + (name) => !isStagingLockFile(name), + ); + expect(tokensBefore).toHaveLength(1); + const liveToken = tokensBefore[0]; + if (liveToken === undefined) throw new Error("No live staging token found"); + + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const prepared = yield* store.prepare({ ...request, key }); + expect(prepared.outcome).toBe("downloaded"); + + const tokensAfter = yield* fs.readDirectory(stagingRoot); + expect(tokensAfter).toEqual(expect.arrayContaining([liveToken, `${liveToken}.lock`])); + expect(yield* fs.exists(`${stagingRoot}/${liveToken}/content`)).toBe(true); + + yield* killAndAwaitExit(handle); + }), + ), + 20_000, + ); +}); + +describe("pins", () => { + it.live( + "keeps a generation pinned for one in-process consumer while another releases, and retires it only once both are gone", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-pin-inprocess-", + }); + const key = "database/postgres-pin-inprocess"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + + const firstScope = yield* Scope.make(); + const firstUse = yield* store.use({ ...request, key }).pipe(Scope.provide(firstScope)); + const secondScope = yield* Scope.make(); + const secondUse = yield* store.use({ ...request, key }).pipe(Scope.provide(secondScope)); + expect(firstUse.path).toBe(secondUse.path); + + yield* ageLockFile(fs, firstUse.lockPath); + const altStore = yield* makeArtifactStore({ + cacheRoot: root, + source: sourceWritingAlt(), + }); + + // Sweeping while both consumers hold the pin: busy against this process's own open + // connection, so the stale generation survives untouched. + yield* altStore.prepare({ ...request, key }); + expect(yield* fs.exists(firstUse.path)).toBe(true); + + yield* Scope.close(firstScope, Exit.void); + yield* altStore.prepare({ ...request, key }); + expect(yield* fs.exists(firstUse.path)).toBe(true); + + yield* Scope.close(secondScope, Exit.void); + yield* altStore.prepare({ ...request, key }); + expect(yield* fs.exists(firstUse.path)).toBe(false); + expect(yield* fs.exists(firstUse.lockPath)).toBe(true); + }), + ), + ); + + it.live( + "a running native workload's generation survives a concurrent retirement sweep, across processes, until it releases", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-pin-cross-process-", + }); + const key = "database/postgres-pin-cross-process"; + const handle = yield* spawnFixture(["use", root, key]); + const line = yield* firstLine(handle); + const match = /^used:(\w+):(.+):(.+)$/.exec(line); + if (match === null) throw new Error(`Unparseable fixture output: ${line}`); + const generationPath = match[2]; + const lockPath = match[3]; + if (generationPath === undefined || lockPath === undefined) + throw new Error(`Unparseable fixture output: ${line}`); + + yield* ageLockFile(fs, lockPath); + const altStore = yield* makeArtifactStore({ + cacheRoot: root, + source: sourceWritingAlt(), + }); + yield* altStore.prepare({ ...request, key }); + expect(yield* fs.exists(generationPath)).toBe(true); + + yield* killAndAwaitExit(handle); + yield* ageLockFile(fs, lockPath); + yield* altStore.prepare({ ...request, key }); + expect(yield* fs.exists(generationPath)).toBe(false); + expect(yield* fs.exists(lockPath)).toBe(true); + }), + ), + 20_000, + ); + + it.live("a retirement sweep that runs between prepare and use cannot break the launch", () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-sweep-between-", + }); + const key = "database/postgres-sweep-between"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const prepared = yield* store.prepare({ ...request, key }); + expect(prepared.outcome).toBe("downloaded"); + + // Ages the generation past the retention window: ahead-of-time `prepare` only pins + // itself for its own duration (see `ArtifactStore.prepare`), so by now it is unpinned + // and genuinely eligible for retirement, unlike the pinned scenarios covered above. + yield* ageLockFile(fs, prepared.lockPath); + + // A sibling digest under the same key forces a sweep as a side effect. + const altStore = yield* makeArtifactStore({ cacheRoot: root, source: sourceWritingAlt() }); + yield* altStore.prepare({ ...request, key }); + expect(yield* fs.exists(prepared.path)).toBe(false); + expect(yield* fs.exists(prepared.lockPath)).toBe(true); + + // `use` transparently re-prepares the now-retired generation instead of failing the + // launch, publishing it fresh again at the same content-addressed path. + const used = yield* Effect.scoped(store.use({ ...request, key })); + expect(used.outcome).toBe("downloaded"); + expect(used.path).toBe(prepared.path); + expect(yield* fs.exists(`${used.path}/bin/postgres`)).toBe(true); }), ), ); + + it.live( + "the native launcher holds its own pin independent of the owner, releasing it only once it reports the workload's exit", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-launcher-pin-", + }); + const key = "database/postgres-launcher-pin"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const prepared = yield* store.prepare({ ...request, key }); + + const environment = yield* Environment.confine(fs, path, `${root}/home`); + const processScope = yield* Scope.make(); + yield* spawnNativeProcess( + { + executable: "/bin/sleep", + args: ["30"], + environment, + artifactLockPath: prepared.lockPath, + gracefulStopSignal: "SIGTERM", + gracefulStopTimeout: "1 second", + }, + defaultNativeProcessLauncher(), + ).pipe(Scope.provide(processScope)); + + yield* ageLockFile(fs, prepared.lockPath); + const altStore = yield* makeArtifactStore({ + cacheRoot: root, + source: sourceWritingAlt(), + }); + yield* altStore.prepare({ ...request, key }); + expect(yield* fs.exists(prepared.path)).toBe(true); + + yield* Scope.close(processScope, Exit.void); + yield* ageLockFile(fs, prepared.lockPath); + yield* altStore.prepare({ ...request, key }); + expect(yield* fs.exists(prepared.path)).toBe(false); + }), + ), + 20_000, + ); + + it.live( + "keeps the generation pinned until every process-group member is gone, not only the direct child", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-launcher-group-", + }); + const key = "database/postgres-launcher-group"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const prepared = yield* store.prepare({ ...request, key }); + + const environment = yield* Environment.confine(fs, path, `${root}/home`); + const pidPath = path.join(root, "descendant.pid"); + const readyPath = path.join(root, "ready"); + const goPath = path.join(root, "go"); + const processScope = yield* Scope.make(); + // The descendant ignores SIGTERM and records its own pid, then the leader signals + // readiness and waits for the test's own release file before exiting: only the + // group-wide SIGKILL in the launcher's exit handler can end the descendant. + yield* spawnNativeProcess( + { + executable: "/bin/sh", + args: [ + "-c", + // `$!` (captured by the leader right after backgrounding) is the descendant's + // real pid; `$$` read from inside the backgrounded subshell is unreliable here, + // since this shell reuses the leader's own pid for it. + '(trap "" TERM; while true; do sleep 1; done) & echo $! > "$1/descendant.pid"; ' + + 'touch "$1/ready"; while [ ! -f "$1/go" ]; do sleep 0.05; done', + "sh", + root, + ], + environment, + artifactLockPath: prepared.lockPath, + }, + defaultNativeProcessLauncher(), + ).pipe(Scope.provide(processScope)); + + const pollUntil = ( + check: Effect.Effect, + satisfied: (value: A) => boolean, + guard: Duration.Input, + ) => + check.pipe( + Effect.filterOrFail(satisfied, () => "not yet" as const), + Effect.retry({ + schedule: Schedule.spaced("20 millis").pipe(Schedule.upTo({ duration: guard })), + while: () => true, + }), + ); + const isPinBusy = Effect.scoped( + acquireLock(prepared.lockPath, "existing").pipe( + Effect.flatMap(takeExclusiveLock), + Effect.as(false), + Effect.catch((error) => Effect.succeed(isBusy(error))), + ), + ); + const isESRCH = (cause: unknown): boolean => + typeof cause === "object" && + cause !== null && + "code" in cause && + cause.code === "ESRCH"; + class ProcessProbeError extends Data.TaggedError("ProcessProbeError")<{ + readonly cause: unknown; + }> {} + // `true` means still alive; ESRCH resolves to `false` (terminated); any other error + // (an unexpected probe failure) propagates instead of being read either way. + const probeAlive = (pid: number) => + Effect.try({ + try: (): boolean => { + process.kill(pid, 0); + return true; + }, + catch: (cause) => new ProcessProbeError({ cause }), + }).pipe( + Effect.catch((error) => + isESRCH(error.cause) ? Effect.succeed(false) : Effect.fail(error), + ), + ); + /** + * Totalizes two separately-timed probes against the race where `kill(pid, 0)` finds the + * descendant alive but it vanishes before `ps` runs moments later: `ps` finding nothing + * (or failing outright) re-checks with `kill` rather than reading empty output as "still + * running". An unexpected (non-ESRCH) error from either probe fails the check. + */ + const isDescendantTerminated = (pid: number) => + Effect.gen(function* () { + if (!(yield* probeAlive(pid))) return true; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const { output, exitCode } = yield* Effect.scoped( + Effect.gen(function* () { + const inspection = yield* ChildProcess.make( + "ps", + ["-o", "stat=", "-p", String(pid)], + { + stdin: "ignore", + stdout: "pipe", + stderr: "ignore", + }, + ); + const [stdout, exitCode] = yield* Effect.all( + [ + inspection.stdout.pipe(Stream.decodeText, Stream.mkString), + inspection.exitCode, + ], + { concurrency: 2 }, + ); + return { output: stdout.trim(), exitCode: Number(exitCode) }; + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)), + ); + if (exitCode === 0 && output.length > 0) return output.startsWith("Z"); + return !(yield* probeAlive(pid)); + }); + + // Readiness (the descendant's pid recorded, the leader waiting on `go`) is an + // observable file, not a fixed delay: polls its existence, bounded by a guard timeout. + yield* pollUntil(fs.exists(readyPath), (ready) => ready, "5 seconds"); + const descendantPid = Number((yield* fs.readFileString(pidPath)).trim()); + + expect(yield* isPinBusy).toBe(true); + + yield* fs.writeFileString(goPath, ""); + + // Polls the lock's own availability, not a fixed delay, bounded by a guard timeout. + yield* pollUntil(isPinBusy, (busy) => !busy, "10 seconds"); + + // Correlates the lock's release with the descendant's own termination: a launcher that + // released the pin before actually killing the group would still show it running. + expect(yield* isDescendantTerminated(descendantPid)).toBe(true); + + yield* Scope.close(processScope, Exit.void); + }), + ), + 20_000, + ); +}); + +describe("retirement across release keys", () => { + it.live( + "retires an older release key's stale, unlocked generation when a newer release key is prepared", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-cross-key-", + }); + // Keys embed a release version (`Artifacts.ts`'s `artifactKey`), so a CLI upgrade + // resolves a different key entirely rather than a new digest under the same one. + const olderKey = "database/postgres/17.0.0-r0/darwin-arm64"; + const newerKey = "database/postgres/17.0.1-r0/darwin-arm64"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const older = yield* store.prepare({ ...request, key: olderKey }); + expect(older.outcome).toBe("downloaded"); + yield* ageLockFile(fs, older.lockPath); + + yield* store.prepare({ ...request, key: newerKey }); + + expect(yield* fs.exists(older.path)).toBe(false); + expect(yield* fs.exists(older.lockPath)).toBe(true); + }), + ), + ); }); diff --git a/packages/stack/src/preparation/slim-services.integration.test.ts b/packages/stack/src/preparation/slim-services.integration.test.ts index c321aae3c1..5d18634562 100644 --- a/packages/stack/src/preparation/slim-services.integration.test.ts +++ b/packages/stack/src/preparation/slim-services.integration.test.ts @@ -668,7 +668,10 @@ describe("slim-services artifact source", () => { const fetcher = serving({ "example.test": { archive, manifest: demoManifest } }); const failed = yield* withFetch(fetcher, store.prepare(request).pipe(Effect.exit)); expect(Exit.isFailure(failed)).toBe(true); - expect(yield* fs.exists(`${root}/demo/v1`)).toBe(false); + const entries = yield* fs + .readDirectory(`${root}/demo/v1`) + .pipe(Effect.orElseSucceed(() => [])); + expect(entries.filter((name) => /^[0-9a-f]{64}$/u.test(name))).toEqual([]); current = demo; const prepared = yield* withFetch(fetcher, store.prepare(request)); expect(yield* fs.readFileString(`${prepared.path}/bin/demo`)).toBe("demo"); @@ -823,43 +826,40 @@ describe("native artifact catalog", () => { ), ); - it.live("keys the native cache by release version and ignores a legacy upstream entry", () => - Effect.scoped( - Effect.gen(function* () { - const resolved = yield* resolveArtifact({ service: "rest" }); - const fs = yield* FileSystem.FileSystem; - const cacheRoot = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-cache-" }); - const requested: string[] = []; - const seed = (version: string, content: string) => - makeArtifactStore({ cacheRoot, source: fixtureSource(content) }).pipe( - Effect.flatMap((store) => - withFetch( - serving({}, requested), + it.live( + "keys the native cache by content: a fixture seeded under the real pinned digest is used, and an unpinned digest is never requested", + () => + Effect.scoped( + Effect.gen(function* () { + const resolved = yield* resolveArtifact({ service: "rest" }); + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-cache-" }); + const requested: string[] = []; + const digest = resolved.natives["linux-amd64"].archive; + const seeded = yield* withFetch( + serving({}, requested), + makeArtifactStore({ + cacheRoot, + source: { ...fixtureSource("pinned"), checksum: () => Effect.succeed(digest) }, + }).pipe( + Effect.flatMap((store) => store.prepare({ - key: `slim-services/postgrest/${version}/linux-amd64`, + key: `slim-services/postgrest/${resolved.releaseVersion}/linux-amd64`, requiredRuntimePaths: resolved.requiredRuntimePaths, executablePath: resolved.executablePath, }), ), ), ); - const prepare = withFetch( - serving({}, requested), - prepareNativeArtifact({ service: "rest" }, cacheRoot, linux), - ); - - yield* seed(resolved.version, "legacy"); - expect(Exit.isFailure(yield* prepare.pipe(Effect.exit))).toBe(true); - expect(requested).not.toEqual([]); - - const seeded = yield* seed(resolved.releaseVersion, "pinned"); - requested.length = 0; - const prepared = yield* prepare; - expect(prepared.root).toBe(seeded.path); - expect(yield* fs.readFileString(prepared.executable)).toBe("pinned"); - expect(requested).toEqual([]); - }).pipe(Effect.provide(NodeServices.layer)), - ), + const prepared = yield* withFetch( + serving({}, requested), + prepareNativeArtifact({ service: "rest" }, cacheRoot, linux), + ); + expect(prepared.root).toBe(seeded.path); + expect(yield* fs.readFileString(prepared.executable)).toBe("pinned"); + expect(requested).toEqual([]); + }).pipe(Effect.provide(NodeServices.layer)), + ), ); it("catalog has no placeholder pins", () => { diff --git a/packages/stack/src/runtime/NativeProcess.ts b/packages/stack/src/runtime/NativeProcess.ts index bd078f7001..e4f6650a5b 100644 --- a/packages/stack/src/runtime/NativeProcess.ts +++ b/packages/stack/src/runtime/NativeProcess.ts @@ -18,6 +18,12 @@ export interface NativeProcessSpec { /** The owned HOME/TMPDIR/XDG/Deno directories this workload is confined to. */ readonly environment: Environment.NativeEnvironment; readonly cwd?: string; + /** + * The native artifact generation's digest lock file. The launcher takes its own SHARED pin on + * this before spawning the workload, and holds it until the workload's process group has + * exited, including the forced-kill path. + */ + readonly artifactLockPath?: string; readonly stdin?: "ignore" | "pipe"; /** Numeric identity the workload runs as; omitted keeps the launcher's identity. */ readonly uid?: number; @@ -92,6 +98,7 @@ const encodeSpec = (spec: NativeProcessSpec, env: Readonly { + try { + writeSync(2, message); + } catch {} +}; interface LaunchSpec { readonly executable: string; @@ -14,6 +24,7 @@ interface LaunchSpec { readonly cwd?: string; readonly uid?: number; readonly gid?: number; + readonly artifactLockPath?: string; readonly gracefulStopSignal?: "SIGTERM" | "SIGINT"; readonly gracefulStopTimeoutMs?: number; } @@ -25,6 +36,7 @@ const LaunchSpecSchema = Schema.Struct({ cwd: Schema.optionalKey(Schema.String), uid: Schema.optionalKey(Schema.Int.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(0)))), gid: Schema.optionalKey(Schema.Int.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(0)))), + artifactLockPath: Schema.optionalKey(Schema.String), gracefulStopSignal: Schema.optionalKey(Schema.Literals(["SIGTERM", "SIGINT"])), gracefulStopTimeoutMs: Schema.optionalKey( Schema.Finite.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(0))), @@ -48,6 +60,7 @@ const decodeSpec = (bytes: Buffer): LaunchSpec | undefined => { env: value.env, uid: value.uid, gid: value.gid, + artifactLockPath: value.artifactLockPath, ...(value.gracefulStopSignal === undefined ? {} : { gracefulStopSignal: value.gracefulStopSignal }), @@ -58,6 +71,32 @@ const decodeSpec = (bytes: Buffer): LaunchSpec | undefined => { } }; +/** + * Takes this launcher's own SHARED pin on the workload's artifact generation before it spawns, + * independent of whatever pin its owner process holds: the launcher outlives a dead owner, so its + * own kernel lock is what keeps a concurrent retirement sweep from touching a generation whose + * workload is still running. There is no heartbeat; the lock's mtime is touched once, here. + */ +const pinArtifactGeneration = (lockPath: string): (() => void) | undefined => { + try { + const connection = new DatabaseSync(lockPath); + takeSharedLockSync(connection); + // oxlint-disable-next-line effecttsgo/global-date -- this standalone process has no Effect Clock service to source "now" from. + const now = new Date(); + utimesSync(lockPath, now, now); + return () => { + try { + connection.close(); + } catch { + // The workload has already exited; a failure to close is not actionable here. + } + }; + } catch (error) { + report(`Unable to pin the native artifact generation: ${String(error)}\n`); + return undefined; + } +}; + const inheritedEnvironmentNames = [ "PATH", "HOME", @@ -115,7 +154,9 @@ export const runNativeLauncher = (): void => { // before this launcher exits while preserving the workload's exit code. process.kill(-(child?.pid ?? process.pid), signal); } catch { - child?.kill(signal); + try { + child?.kill(signal); + } catch {} } }; @@ -204,6 +245,66 @@ export const runNativeLauncher = (): void => { } else { specGracefulStopSignal = spec.gracefulStopSignal; specGracefulStopTimeoutMs = spec.gracefulStopTimeoutMs; + // Taken before spawning and released only after the workload's group has been sent SIGKILL, + // on every exit path below, so a concurrent retirement sweep never races a running workload. + const releasePin = + spec.artifactLockPath === undefined + ? undefined + : pinArtifactGeneration(spec.artifactLockPath); + if (spec.artifactLockPath !== undefined && releasePin === undefined) { + // The pin could not be taken: never spawn a workload whose generation isn't protected. + // Reporting nothing on fd5 is the same failure the owner already handles when the + // launcher never writes a valid process group. + process.exit(127); + return; + } + /** + * ESRCH (no such process group) counts as success: the group is already gone. A real failure + * (for example EPERM) must never be treated as confirmation that it is. + */ + const killGroupForExit = (pgid: number): boolean => { + if (process.platform === "win32") return true; + try { + process.kill(-pgid, "SIGKILL"); + return true; + } catch (error) { + return ( + typeof error === "object" && error !== null && "code" in error && error.code === "ESRCH" + ); + } + }; + // Once SIGKILL has reached every group member, none can run user code again, and unlinking + // this generation's files afterward cannot affect an fd or mapping a member already holds + // open: releasing the pin right after the kill, without waiting to confirm each member has + // actually been reaped, is safe. A real kill failure must never reach `afterRelease`, since + // exiting closes this process's own fd and reclaims the advisory lock regardless of whether + // `releasePin` was ever called: staying alive and retrying is the only way to keep it held. + const releaseAfterGroupKill = (afterRelease: () => void): void => { + const pgid = child?.pid; + if (releasePin === undefined) { + afterRelease(); + return; + } + if (pgid === undefined) { + releasePin(); + afterRelease(); + return; + } + const attempt = (reported: boolean): void => { + if (killGroupForExit(pgid)) { + releasePin(); + afterRelease(); + return; + } + if (!reported) + report( + "Unable to confirm the native workload's process group is gone; keeping its artifact generation pinned and retrying\n", + ); + // oxlint-disable-next-line effecttsgo/global-timers -- this standalone process has no Effect runtime to schedule through. + setTimeout(() => attempt(true), 1_000); + }; + attempt(false); + }; child = spawn(spec.executable, [...spec.args], { cwd: spec.cwd, env: { ...Effect.runSync(inheritedEnvironment), ...spec.env }, @@ -216,23 +317,26 @@ export const runNativeLauncher = (): void => { writeSync(5, `${child.pid ?? 0}\n`); } catch { terminateWorkloadGroup("SIGKILL"); - process.exit(127); + releaseAfterGroupKill(() => process.exit(127)); + return; } child.on("error", (error) => { - writeSync(2, `Native workload failed to start: ${error.message}\n`); - process.exit(127); + report(`Native workload failed to start: ${error.message}\n`); + releaseAfterGroupKill(() => process.exit(127)); }); child.on("exit", (code, signal) => { childExited = true; terminateWorkloadGroup("SIGKILL"); - if (ownerLossGraceful) { + releaseAfterGroupKill(() => { + if (ownerLossGraceful) { + process.exit(code ?? 1); + return; + } + ownerPipe.destroy(); + if (!gracefulForwarded && signal !== null) + report(`Native workload exited due to signal ${signal}\n`); process.exit(code ?? 1); - return; - } - ownerPipe.destroy(); - if (!gracefulForwarded && signal !== null) - writeSync(2, `Native workload exited due to signal ${signal}\n`); - process.exit(code ?? 1); + }); }); } }; diff --git a/packages/stack/src/runtime/postgres-user.ts b/packages/stack/src/runtime/postgres-user.ts index 9d0c3f5c00..bb38c05c10 100644 --- a/packages/stack/src/runtime/postgres-user.ts +++ b/packages/stack/src/runtime/postgres-user.ts @@ -255,6 +255,25 @@ const allowTraverse = Effect.fn("NativePostgresUser.allowTraverse")(function* ( } }); +/** + * A directory inside a published artifact generation keeps the archive's own mode for life, so + * the step-down handover checks for the traverse bit (o+x) there instead of granting it: granting + * it would mutate an already-published, content-addressed generation. + */ +const requireTraverse = Effect.fn("NativePostgresUser.requireTraverse")(function* ( + directories: ReadonlyArray, +) { + const fs = yield* FileSystem.FileSystem; + for (const directory of directories) { + const { mode } = yield* fs.stat(directory); + if ((mode & 0o001) === 0) + return yield* new ServiceError({ + operation: "launch", + message: `${directory} is missing its traverse bit (o+x); re-prepare the artifact`, + }); + } +}); + const requireOwnedByRootOr = Effect.fn("NativePostgresUser.requireOwnedByRootOr")(function* ( user: PasswdEntry, target: string, @@ -330,7 +349,9 @@ const chownRecursive = Effect.fn("NativePostgresUser.chownRecursive")(function* * native `environmentHome` (if any) to the PostgreSQL user, each with its own `chown -R -P` call * so a nested directory is never also listed as a separate, redundant target. `runtimeRoot` and its * ancestors only gain traverse, so the step-down user reaches its socket directory under a - * restrictive configured base without write access to any of them. + * restrictive configured base without write access to any of them. An ancestor inside the + * published generation (the bundle root and anything between it and the executable or getkey) + * is only required to already have traverse, never granted it: that generation is immutable. */ export const handOverNativePostgresFiles = Effect.fn("NativePostgresUser.handOverFiles")(function* ( user: PasswdEntry, @@ -366,9 +387,17 @@ export const handOverNativePostgresFiles = Effect.fn("NativePostgresUser.handOve }); yield* chownRecursive(user, paths.environmentHome); } + const insideGeneration = (candidate: string) => + candidate === paths.bundleRoot || candidate.startsWith(`${paths.bundleRoot}${path.sep}`); + const executableAncestors = ancestorsOf(path, path.dirname(paths.executable)); + const getkeyAncestors = hasGetkey ? ancestorsOf(path, path.dirname(getkey)) : []; + yield* requireTraverse([ + ...executableAncestors.filter(insideGeneration), + ...getkeyAncestors.filter(insideGeneration), + ]); yield* allowTraverse(user, [ ...ancestorsOf(path, paths.runtimeRoot), - ...ancestorsOf(path, path.dirname(paths.executable)), - ...(hasGetkey ? ancestorsOf(path, path.dirname(getkey)) : []), + ...executableAncestors.filter((candidate) => !insideGeneration(candidate)), + ...getkeyAncestors.filter((candidate) => !insideGeneration(candidate)), ]); }); diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index 17760fd800..7ae260d2e3 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -26,6 +26,7 @@ import { prepareNativeArtifact, postgresVersion, resolveArtifact, + useNativeArtifact, type PreparedNativeArtifact, } from "../Artifacts.ts"; import { @@ -386,7 +387,7 @@ const nativeProcess = ( readonly rootKeyPath: string; }, settings: ReadonlyArray, - context: ServiceInstanceContext, + scope: Scope.Closeable, stackId: string, instanceId: string, spawner: ChildProcessSpawnerService["Service"], @@ -396,6 +397,7 @@ const nativeProcess = ( spawnNativeProcess( { executable: artifact.executable, + artifactLockPath: artifact.lockPath, ...(user === undefined ? {} : { uid: user.uid, gid: user.gid, cwd: "/" }), args: [ "-D", @@ -424,7 +426,7 @@ const nativeProcess = ( defaultNativeProcessLauncher(), { stackId, workloadId: instanceId }, ).pipe( - Scope.provide(context.scope), + Scope.provide(scope), Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), Effect.mapError((cause) => errorFor("launch", cause)), ); @@ -450,7 +452,6 @@ export const makeDatabase = ( const client = yield* HttpClient.HttpClient; const logs = yield* PubSub.sliding(256); const endpoint = yield* Ref.make(undefined); - const prepared = yield* Ref.make>(new Map()); if (!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/u.test(String(options.stackId))) return yield* databaseError("identity", "Invalid stack id"); // Ownership is by location: instanceRoot is owned simply by being under options.root, with no @@ -602,15 +603,14 @@ export const makeDatabase = ( }), ); if (options.runtime === "native") + // Ahead-of-time warm-up only: downloads and publishes the generation but pins nothing. + // `launch` resolves (or prepares) and pins its own copy right before it spawns. return prepareNativeArtifact({ service: "database", version }, options.cacheRoot).pipe( Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path), Effect.provideService(Crypto.Crypto, crypto), Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), Effect.provideService(HttpClient.HttpClient, client), - Effect.tap((artifact) => - Ref.update(prepared, (map) => new Map(map).set(version, artifact)), - ), Effect.asVoid, Effect.mapError((cause) => errorFor("prepare", cause)), ); @@ -835,9 +835,25 @@ export const makeDatabase = ( yield* fs .writeFileString(hbaPath, NATIVE_HBA_RULES, { mode: 0o600 }) .pipe(Effect.mapError((cause) => errorFor("launch", cause))); - const artifact = (yield* Ref.get(prepared)).get(config.version); - if (artifact === undefined) - return yield* errorFor("launch", `Artifact ${config.version} was not prepared`); + // `context.scope` finalizes its direct children in parallel (it is forked "parallel" + // in Service.ts), so registering the pin there directly would race its release + // against the native process's own cleanup. A "sequential" child scope finalizes + // LIFO instead: the pin is registered on it first, and the native process below is + // also scoped to it (not to `context.scope`), so closing it always runs the + // process's cleanup before releasing the pin. + const launchScope = yield* Scope.fork(context.scope, "sequential"); + const artifact = yield* useNativeArtifact( + { service: "database", version: config.version }, + options.cacheRoot, + ).pipe( + Scope.provide(launchScope), + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(Crypto.Crypto, crypto), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService(HttpClient.HttpClient, client), + Effect.mapError((cause) => errorFor("launch", cause)), + ); if (stepDownUser !== undefined) yield* asRoot( handOverNativePostgresFiles(stepDownUser, { @@ -856,7 +872,7 @@ export const makeDatabase = ( config, { dataPath, socketPath, hbaPath, rootKeyPath }, settings, - context, + launchScope, String(options.stackId), options.instanceId, spawner, diff --git a/packages/stack/src/services/ProcessRecipe.integration.test.ts b/packages/stack/src/services/ProcessRecipe.integration.test.ts index e5b41f8557..77bc573803 100644 --- a/packages/stack/src/services/ProcessRecipe.integration.test.ts +++ b/packages/stack/src/services/ProcessRecipe.integration.test.ts @@ -495,15 +495,19 @@ const nativeFixtureArtifact = Effect.fn(function* ( : undefined; if (target === undefined) return yield* Effect.fail(`Unsupported test platform: ${platformName}`); - const { releaseVersion } = yield* resolveArtifact({ service: artifact.service }); + const { releaseVersion, natives } = yield* resolveArtifact({ service: artifact.service }); const request: ArtifactRequest = { key: `slim-services/${artifact.name}/${releaseVersion}/${target}`, requiredRuntimePaths: Object.keys(artifact.files), executablePath: artifact.executablePath, }; const fs = yield* FileSystem.FileSystem; + // The generation directory is content-addressed by this digest: it must match what the real + // artifact source resolves for this platform, or this fixture is invisible to consumers that + // resolve the real catalog source for the same key. + const digest = natives[target].archive; const source: ArtifactSource = { - checksum: () => Effect.succeed("0".repeat(64)), + checksum: () => Effect.succeed(digest), materialize: (_entry, destination) => Effect.gen(function* () { yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); diff --git a/packages/stack/src/services/ProcessRecipe.ts b/packages/stack/src/services/ProcessRecipe.ts index 0b54af3536..8d567327c0 100644 --- a/packages/stack/src/services/ProcessRecipe.ts +++ b/packages/stack/src/services/ProcessRecipe.ts @@ -21,7 +21,12 @@ import { ChildProcessSpawner } from "effect/unstable/process"; import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import * as Net from "node:net"; -import { prepareNativeArtifact, resolveArtifact, type ServiceKind } from "../Artifacts.ts"; +import { + prepareNativeArtifact, + resolveArtifact, + useNativeArtifact, + type ServiceKind, +} from "../Artifacts.ts"; import * as Environment from "../namespace/Environment.ts"; import { containerInstancePath, destroyOwnedRoot, type BorrowedPath } from "../namespace/Paths.ts"; import { accepts } from "../Ports.ts"; @@ -440,8 +445,6 @@ export const makeProcessRecipe = spec: ProcessRecipeSpec, ): Effect.Effect> => Effect.gen(function* () { - const prepared = yield* Ref.make(undefined); - const preparedRoot = yield* Ref.make(undefined); const endpoints = yield* Ref.make>(new Map()); const logs = yield* PubSub.sliding(256); // Ownership is by location: instanceRoot is owned simply by being under options.root, created @@ -470,7 +473,9 @@ export const makeProcessRecipe = version: candidate.version, }).pipe(Effect.mapError((cause) => serviceError("prepare", cause))); if (options.runtime === "native") { - const artifact = yield* prepareNativeArtifact( + // Ahead-of-time warm-up only: downloads and publishes the generation but pins nothing. + // `launch` resolves (or prepares) and pins its own copy right before it spawns. + yield* prepareNativeArtifact( { service: candidate.service, version: candidate.version }, options.cacheRoot, options.platform, @@ -482,8 +487,6 @@ export const makeProcessRecipe = Effect.provideService(HttpClient.HttpClient, deps.client), Effect.mapError((cause) => serviceError("prepare", cause)), ); - yield* Ref.set(prepared, artifact.executable); - yield* Ref.set(preparedRoot, artifact.root); } else { if (deps.container === undefined) return yield* serviceError("prepare", "Container runtime unavailable"); @@ -502,12 +505,27 @@ export const makeProcessRecipe = ([name]) => spec.enabledPort === undefined || spec.enabledPort(context.config, name), ); if (options.runtime === "native") { - const executable = yield* Ref.get(prepared); - const artifactRoot = yield* Ref.get(preparedRoot); - if (executable === undefined) - return yield* serviceError("launch", "Artifact was not prepared"); - if (artifactRoot === undefined) - return yield* serviceError("launch", "Artifact root was not prepared"); + // `context.scope` finalizes its direct children in parallel (it is forked "parallel" in + // Service.ts), so registering the pin there directly would race its release against the + // spawned processes' own cleanup. A "sequential" child scope finalizes LIFO instead: the + // pin is registered on it first, every native process scope below forks from it (not from + // `context.scope`), so closing it always runs their cleanup before releasing the pin. + const launchScope = yield* Scope.fork(context.scope, "sequential"); + const artifact = yield* useNativeArtifact( + { service: context.config.service, version: context.config.version }, + options.cacheRoot, + options.platform, + ).pipe( + Scope.provide(launchScope), + Effect.provideService(FileSystem.FileSystem, deps.fs), + Effect.provideService(Path.Path, deps.path), + Effect.provideService(Crypto.Crypto, deps.crypto), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, deps.spawner), + Effect.provideService(HttpClient.HttpClient, deps.client), + Effect.mapError((cause) => serviceError("launch", cause)), + ); + const executable = artifact.executable; + const artifactRoot = artifact.root; const environment = yield* Environment.confine(deps.fs, deps.path, environmentRoot).pipe( Effect.mapError((cause) => serviceError("launch", cause)), ); @@ -542,7 +560,7 @@ export const makeProcessRecipe = } | undefined; if (spec.startupCommands.length > 0) { - const startupScope = yield* Scope.fork(context.scope, "sequential"); + const startupScope = yield* Scope.fork(launchScope, "sequential"); const reservation = spec.nativeStartupEnv === undefined ? undefined @@ -565,6 +583,7 @@ export const makeProcessRecipe = env: command.env, environment, cwd: artifactRoot, + artifactLockPath: artifact.lockPath, }, defaultNativeProcessLauncher(), { @@ -597,7 +616,7 @@ export const makeProcessRecipe = readonly portScope: Scope.Closeable; readonly endpoints: ReadonlyMap; }) { - const scope = yield* Scope.fork(context.scope, "sequential"); + const scope = yield* Scope.fork(launchScope, "sequential"); const reservation = held ?? (yield* reserveEndpoints(scope)); const selected = reservation.endpoints; const args = yield* spec.args(context.config, selected, { @@ -612,6 +631,7 @@ export const makeProcessRecipe = args, env, environment, + artifactLockPath: artifact.lockPath, gracefulStopSignal: "SIGTERM", gracefulStopTimeout: "5 seconds", }, diff --git a/packages/stack/tests/artifact-store-fixture.ts b/packages/stack/tests/artifact-store-fixture.ts new file mode 100644 index 0000000000..3fe0f3b006 --- /dev/null +++ b/packages/stack/tests/artifact-store-fixture.ts @@ -0,0 +1,93 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { Console, Crypto, Effect, FileSystem, Layer } from "effect"; +import { + makeArtifactStore, + type ArtifactRequest, + type ArtifactSource, +} from "../src/preparation/ArtifactStore.ts"; +import { ArtifactIntegrityError, PreparationError } from "../src/preparation/Errors.ts"; +import { verifySha256 } from "../src/preparation/Integrity.ts"; + +const mapMaterializeError = (effect: Effect.Effect) => + effect.pipe( + Effect.mapError((cause) => + cause instanceof PreparationError || cause instanceof ArtifactIntegrityError + ? cause + : new PreparationError({ + message: `materialization failed: ${cause instanceof Error ? cause.message : String(cause)}`, + cause, + }), + ), + ); + +/** + * Cross-process fixture for `artifacts.integration.test.ts`: real preparer/pin races need real + * separate processes, not fibers in one process. Modes: + * - `prepare`: ahead-of-time `store.prepare`, prints `prepared:::`, exits. + * - `use`: scoped `store.use`, prints `used:::`, then blocks forever so + * the parent controls the pin's release by killing this process. + * - `stall`: `store.prepare` whose materialize prints `started` once extraction has begun, then + * blocks forever, so the parent can synchronize a SIGKILL precisely mid-extraction without + * polling. + */ +const [mode, cacheRoot, key] = process.argv.slice(2); +if (mode === undefined || cacheRoot === undefined || key === undefined) + throw new Error("Usage: artifact-store-fixture "); + +const archive = new TextEncoder().encode("archive"); +const archiveSha256 = "0eb3e36bfb24dcd9bb1d1bece1531216b59539a8fde17ee80224af0653c92aa3"; + +const request: ArtifactRequest = { + key, + requiredRuntimePaths: ["bin/postgres", "etc/postgres.conf"], + executablePath: "bin/postgres", +}; + +const fastSource: ArtifactSource = { + checksum: () => Effect.succeed(archiveSha256), + materialize: (_request, destination, expectedSha256) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); + yield* fs.makeDirectory(`${destination}/etc`, { recursive: true }); + yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres", { mode: 0o755 }); + yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config", { mode: 0o644 }); + const crypto = yield* Crypto.Crypto; + yield* verifySha256(archive, expectedSha256).pipe( + Effect.provideService(Crypto.Crypto, crypto), + ); + }).pipe(mapMaterializeError), +}; + +const stallingSource: ArtifactSource = { + checksum: () => Effect.succeed(archiveSha256), + materialize: (_request, destination) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); + yield* Console.log("started"); + return yield* Effect.never; + }).pipe(mapMaterializeError), +}; + +const program = Effect.scoped( + Effect.gen(function* () { + const store = yield* makeArtifactStore({ + cacheRoot, + source: mode === "stall" ? stallingSource : fastSource, + }); + if (mode === "prepare" || mode === "stall") { + const prepared = yield* store.prepare(request); + yield* Console.log(`prepared:${prepared.outcome}:${prepared.path}:${prepared.sha256}`); + return; + } + if (mode === "use") { + const prepared = yield* store.use(request); + yield* Console.log(`used:${prepared.outcome}:${prepared.path}:${prepared.lockPath}`); + return yield* Effect.never; + } + throw new Error(`Unknown fixture mode: ${mode}`); + }), +).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))); + +await Effect.runPromise(program); From 9b9907768a831661bfcedd79760a27c925af767d Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sun, 4 Oct 2026 02:36:42 +0200 Subject: [PATCH 016/142] chore(stack): keep the pin table helper private to its driver --- packages/stack/src/namespace/drivers/sqlite-pin.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/stack/src/namespace/drivers/sqlite-pin.ts b/packages/stack/src/namespace/drivers/sqlite-pin.ts index 499d8e3ee9..f7d496bede 100644 --- a/packages/stack/src/namespace/drivers/sqlite-pin.ts +++ b/packages/stack/src/namespace/drivers/sqlite-pin.ts @@ -3,12 +3,10 @@ import type { DatabaseSync } from "node:sqlite"; /** - * The one-row table a pin's `SELECT` reads, created lazily so a fresh digest lock file works. - * Checks first and only writes when actually missing: every pin after the first ever one on a - * digest finds the table and row already there, so it never takes a write-intent lock at all, - * which would otherwise transiently conflict with another concurrent reader's own no-op check. + * The one-row table a pin's `SELECT` reads. It writes only when missing, so later pins never take a + * write-intent lock that would conflict with a concurrent reader. */ -export const ensurePinTable = (connection: DatabaseSync): void => { +const ensurePinTable = (connection: DatabaseSync): void => { const hasTable = connection .prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'pin'") .get(); From ee990a017b968a4128821e47b782807f144287fe Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sun, 4 Oct 2026 07:22:00 +0200 Subject: [PATCH 017/142] feat(stack): keep public ports stable through a per-user reservation registry --- .../stack/destroy/SIDE_EFFECTS.md | 9 + .../experimental/stack/list/SIDE_EFFECTS.md | 4 +- .../experimental/stack/start/SIDE_EFFECTS.md | 8 + .../experimental/stack/stop/SIDE_EFFECTS.md | 4 +- packages/stack/README.md | 24 + packages/stack/src/Network.ts | 39 +- packages/stack/src/Owner.ts | 13 +- packages/stack/src/Ports.integration.test.ts | 615 +++++++----------- packages/stack/src/Ports.ts | 384 +++++++---- packages/stack/src/Proxy.ts | 9 +- packages/stack/src/Rpc.ts | 66 +- packages/stack/src/StackNamespace.ts | 1 - .../stack/src/lifetime.integration.test.ts | 6 +- .../namespace/Namespace.integration.test.ts | 3 +- .../stack/src/namespace/PortReservations.ts | 380 +++++++++++ packages/stack/src/namespace/Registry.ts | 23 +- .../stack/src/port-reservation.e2e.test.ts | 465 +++++++++++++ packages/stack/tests/whole-stack/scenarios.ts | 4 +- 18 files changed, 1506 insertions(+), 551 deletions(-) create mode 100644 packages/stack/src/namespace/PortReservations.ts create mode 100644 packages/stack/src/port-reservation.e2e.test.ts diff --git a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md index 15e3f69b12..c8d8170146 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md @@ -41,6 +41,15 @@ uploads are preserved. The independent artifact cache is retained. Communicates with the local owner and selected runtime, with no hosted API calls. Shared routing/settings can read project configuration and profiles. +Destruction releases this stack's rows in the per-user port registry at +`/.supabase/ports.sqlite` (independent of `SUPABASE_HOME`, with no +override) only once cleanup is confirmed complete; a destroy that fails or skips +engine cleanup keeps the stack's reservations, and its ports remain unavailable +to other stacks until a later destroy succeeds. A reservation whose owning +stack's state was deleted without going through destroy (for example a removed +or unmounted state root) is reclaimed lazily by whichever stack next needs that +port. + ## Output, exit codes and telemetry Text prints `Stack destroyed.`, or, when engine cleanup was skipped, diff --git a/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md index c70d13f8f4..6d382077df 100644 --- a/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md @@ -10,7 +10,9 @@ Reads `/stacks//state.json` and probes existin local owner control endpoints. It makes no hosted API calls. Shared command setup may read project configuration for the feature gate and the selected profile for CLI settings. Discovery creates the registry directory if absent and sets its mode to 0700; -no stack state files are written. Telemetry flushes to +no stack state files are written. List does not read the separate per-user public +port registry at `/.supabase/ports.sqlite`; listed entries carry no +port or endpoint information. Telemetry flushes to `/telemetry.json` on success and failure. ## Output diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index d9f6f2543a..a518715048 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -38,6 +38,14 @@ use `$SUPABASE_HOME/cache/stack`, keyed by content digest and retired automatica 30 days. Storage files use the caller-owned project directory `supabase/.temp/stack-uploads//`. Functions preparation may build the project's source. +Public ports are reserved in one SQLite registry per OS user at `/.supabase/ports.sqlite`, +which `$SUPABASE_HOME` does not affect and which has no override. A stopped stack's ports stay +reserved across every other stack's starts, including in a different state root, and starting it +again reuses the same ports. A saved or explicitly requested port already held by another stack, or +occupied by a process outside the registry, fails start naming the conflict instead of picking a +different port; only automatic allocation tries another candidate. On macOS a process can still win +a narrow race against the pre-bind probe before a brand-new listener exists. + For a new stack, `--runtime auto` selects Docker when `docker version` reaches its daemon, then native on Linux x64/arm64 and macOS arm64. The probe is bounded by 10 seconds. Without a reachable engine on other platforms, the command fails and asks the user to start Docker. When auto selection diff --git a/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md index a2bd679794..9df0c80610 100644 --- a/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md @@ -33,7 +33,9 @@ current project/Git identity when no explicit ID is supplied. Shared routing and settings may read project configuration and the selected profile. Communicates only with local owner endpoints; no hosted API requests. Shutdown updates owned runtime resources but preserves persistent instance definitions, data and port -claims. No caller-owned upload files are removed. +reservations in the per-user registry at `/.supabase/ports.sqlite` +(unaffected by `SUPABASE_HOME`); killing the owner process preserves them the +same way. No caller-owned upload files are removed. ## Exit codes and telemetry diff --git a/packages/stack/README.md b/packages/stack/README.md index df18ace9ac..4550fc8077 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -91,6 +91,30 @@ Native PostgreSQL refuses to run as uid 0. When the stack runs as root inside a Native PostgreSQL listens only on its socket and reads a stack-generated HBA file, written to the socket directory on every launch, instead of `PGDATA/pg_hba.conf`. It trusts `supabase_admin`, including through the proxied loopback database port, and requires `scram-sha-256` passwords from every other role. +### Public ports + +Every public port is reserved, before any listener binds it, in one SQLite registry per OS user at +`/.supabase/ports.sqlite`. The home is resolved once per process straight from the OS +user database (`getent passwd ` on Linux, `dscacheutil -q user -a uid ` on macOS), never +`$HOME` or `SUPABASE_HOME`, and with no override; a process cannot steer the registry's location by +setting its own environment. The registry is the only authority on which stack owns a port across +every state root on the machine, so a stopped stack keeps its ports across other +stacks' starts, and a restart reuses the same ports or fails with a `StackError` whose `conflict` +field names the port, the endpoint, and either the live holder (`{ stackId, stateRoot }`) or +`"foreign"` for a process outside the registry. A fixed or previously saved port is never silently +reassigned; only automatic allocation tries another candidate. A reservation is released when +`destroy` confirms the stack has nothing left to clean up, or when deleting an individual service +releases the endpoints only it owned; stopping a stack or killing its owner leaves reservations in +place. A reservation whose owning stack's `state.json` is confirmed gone (`ENOENT`, for example +after deleting its state root) is reclaimed lazily by the next stack that needs its port. + +Known limitations: on macOS, BSD, and Windows, where the kernel allows overlapping binds, a loopback +probe before the real bind narrows but cannot close the race with a foreign process binding in the +same window; two of this user's stacks can never collide, because the registry excludes them. An +unmounted or otherwise temporarily unreachable state root looks identical to a deleted one and can +be reclaimed the same way. The registry does not reserve across OS users; isolation between users' +stacks still relies only on the kernel refusing a second bind. + ## Composition and operation scope Registering a service does not add it to the application composition. For a fresh composition, `composition.supabase` registers the selected recipes and supplies their standard bindings: diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index 3d45e023d9..b4017a60ed 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -1,7 +1,8 @@ import { Context, Data, Effect, Exit, Layer, Ref, Scope, Semaphore } from "effect"; import { DOCKER_HOST_ALIAS } from "./runtime/Container.ts"; +import * as PortReservations from "./namespace/PortReservations.ts"; import * as StackNamespace from "./StackNamespace.ts"; -import { makePorts, PortError } from "./Ports.ts"; +import { makePorts, PortError, probeVacant } from "./Ports.ts"; import { bindTcp, serveTcp, type BackendAddress, type ProxyError } from "./Proxy.ts"; import { makeHttpProxy, type HttpProxy, type HttpRoute } from "./HttpProxy.ts"; @@ -51,6 +52,17 @@ export interface NetworkNamespace { export interface Interface { readonly release: Effect.Effect; + /** + * Marks a stack-wide destroy in progress: every per-instance `NetworkNamespace.release` closes + * its listeners as usual but skips deleting its own reservation rows, so a destroy that later + * turns out uncertain (remaining resource claims) still finds them. Call {@link releaseStack} once + * destroy is confirmed to leave nothing behind, or {@link cancelDestroy} to resume immediate + * per-service deletion (the shape an individual `destroyService` still relies on). + */ + readonly beginDestroy: Effect.Effect; + readonly cancelDestroy: Effect.Effect; + /** Releases every reservation this stack holds, dedicated and shared alike, in one step. */ + readonly releaseStack: Effect.Effect; readonly register: (options: { readonly id: string; readonly endpoints: Readonly>; @@ -70,13 +82,16 @@ const makeNetwork = (options: { readonly stackId: string; readonly runtime: NetworkRuntime; readonly state: StackNamespace.Interface; + readonly platform?: NodeJS.Platform; }) => Effect.gen(function* () { const ports = yield* makePorts(options.state).pipe( Effect.mapError((cause) => errorFor("ports", cause)), ); + const probe = probeVacant(options.platform ?? process.platform); const owner = yield* Scope.Scope; const gate = yield* Semaphore.make(1); + const deferReservations = yield* Ref.make(false); const shared = yield* Ref.make< | { readonly claim: number; @@ -169,15 +184,18 @@ const makeNetwork = (options: { }); return { proxy: current.proxy }; } + yield* probe(key, host, port); return { proxy: yield* makeHttpProxy({ host, port }) }; } if (endpoint.protocol === "http") { + yield* probe(key, host, port); const proxy = yield* makeHttpProxy({ host, port }); yield* proxy.setRoutes([ { id, prefix: "/", target: endpoint.backend }, ]); return { proxy }; } + yield* probe(key, host, port); const listener = yield* bindTcp(host, port); yield* Effect.forkIn( serveTcp(listener, endpoint.backend, `${id}:${name}`).pipe( @@ -287,6 +305,10 @@ const makeNetwork = (options: { "Stop the instance before releasing its endpoints", ); yield* Ref.set(closed, true); + // A stack-wide destroy defers every row deletion to its own success boundary (see + // `beginDestroy`), so an uncertain destroy (remaining resource claims) still finds + // this instance's reservations; an individual `destroyService` still deletes them now. + if (yield* Ref.get(deferReservations)) return; for (const name of Object.keys(endpoints)) { const endpoint = endpoints[name]; if (endpoint?.shared === undefined) @@ -333,7 +355,18 @@ const makeNetwork = (options: { .pipe(Effect.mapError((cause) => errorFor("release", cause))), ), ); - return { register, release: release() } satisfies Interface; + const releaseStack = Effect.fn("Network.releaseStack")(() => + ports + .releaseStack(options.stackId) + .pipe(Effect.mapError((cause) => errorFor("release", cause))), + ); + return { + register, + release: release(), + beginDestroy: Ref.set(deferReservations, true), + cancelDestroy: Ref.set(deferReservations, false), + releaseStack: releaseStack(), + } satisfies Interface; }); export const layer = (options: { readonly stackId: string; readonly runtime: NetworkRuntime }) => @@ -343,4 +376,4 @@ export const layer = (options: { readonly stackId: string; readonly runtime: Net const state = yield* StackNamespace.Service; return yield* makeNetwork({ ...options, state }); }).pipe(Effect.map(Service.of)), - ); + ).pipe(Layer.provide(PortReservations.layer)); diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index cefe2c8142..7444c3f70c 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -767,8 +767,14 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { definitionGate.withPermits(1), Effect.withSpan("Owner.stopNamespace"), ), - destroy: orchestrator.destroyNamespace.pipe( - Effect.andThen(network.release), + destroy: network.beginDestroy.pipe( + // Deferred for the whole of this destroy: every instance's own teardown below closes its + // listeners as usual, but leaves its reservation rows for `releaseStack` to drop together, + // only once destroy is confirmed to leave nothing behind. `ensuring` below resumes + // immediate per-service deletion again on every exit, success, failure, or interruption + // alike, so a service destruction, sweep, or claim-read failure can never leave deferral + // stuck on for a stack that is still otherwise live. + Effect.andThen(orchestrator.destroyNamespace), Effect.andThen(sweep), // A claim reconcile deliberately kept (for example one recorded against a different // daemon) must not be lost to a full deregistration; the stack stays registered so a @@ -776,7 +782,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { Effect.andThen(options.state.readClaims(stackId)), Effect.flatMap((remaining) => remaining.length === 0 - ? options.state.remove(stackId) + ? network.releaseStack.pipe(Effect.andThen(options.state.remove(stackId))) : Effect.gen(function* () { const currentDaemonId = options.engineTarget?.daemonId; const claimsPath = path.join(options.state.root, stackId, Claims.CLAIMS_FILE); @@ -798,6 +804,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { }); }), ), + Effect.ensuring(network.cancelDestroy), definitionGate.withPermits(1), Effect.withSpan("Owner.destroyNamespace"), ), diff --git a/packages/stack/src/Ports.integration.test.ts b/packages/stack/src/Ports.integration.test.ts index 0ec892a955..4457311ea4 100644 --- a/packages/stack/src/Ports.integration.test.ts +++ b/packages/stack/src/Ports.integration.test.ts @@ -1,423 +1,272 @@ -import { NodeServices, NodeSocketServer } from "@effect/platform-node"; +import { NodeFileSystem, NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Cause, Context, Effect, Exit, FileSystem, Layer, Option, Ref, Scope } from "effect"; -import { makePorts, PortError } from "./Ports.ts"; +import { Context, Effect, Exit, FileSystem, Fiber, Layer, Path, Schedule, Scope } from "effect"; +import * as Net from "node:net"; +import { randomUUID } from "node:crypto"; +import { makePorts, PortError, probeVacant } from "./Ports.ts"; +import { bindTcp } from "./Proxy.ts"; +import * as PortReservations from "./namespace/PortReservations.ts"; import * as StackNamespace from "./StackNamespace.ts"; +import { ownerFor } from "../tests/owner-rpc.ts"; + +/** + * Scenarios the public `effect.ts` facade cannot reach deterministically: forcing an + * auto-allocation retry onto a known candidate, which needs the exact first port a real round trip + * picked; a real TCP bind failure, which a probe-equipped `Network.ts` callback would otherwise + * intercept first on a platform where overlapping binds succeed; and a stack-wide destroy's + * deferred row release through a real, in-process `Owner`, whose first attempt a concrete sweep + * failure leaves uncertain. All exercise the real per-user registry and real sockets; see + * `port-reservation.e2e.test.ts` for the end-to-end scenarios, and `Network.integration.test.ts`'s + * "keeps shared routes independent and retains the shared claim" for the shared-listener + * self-reuse case. + */ const makeTestState = (root: string) => Layer.build(StackNamespace.layer({ root })).pipe( Effect.map((context) => Context.get(context, StackNamespace.Service)), ); -const bind = (host: string, port: number) => - NodeSocketServer.make({ host, port }).pipe( - Effect.mapError((cause) => new PortError({ key: "sql", message: "Cannot bind", cause })), - ); - -it.live("retains distinct claims for stopped stacks and rebinds the original public port", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - for (const id of ["first", "second"]) - yield* state.save({ - id, - runtime: "native", - identity: { projectRoot: root, branchContext: "test", stackName: "ports" }, - instances: [], - lifetime: "detached", - composition: { members: [], dependencies: [] }, - ports: [], - }); - const ports = yield* makePorts(state); - const firstScope = yield* Scope.make(); - const request = { stackId: "first", key: "db/sql", host: "127.0.0.1", port: "auto" as const }; - const first = yield* ports - .acquire(request, bind) - .pipe(Effect.provideService(Scope.Scope, firstScope)); - yield* Scope.close(firstScope, Exit.void); - const second = yield* ports.acquire({ ...request, stackId: "second" }, bind); - expect(second.port).not.toBe(first.port); - const reopened = yield* makePorts(yield* makeTestState(root)); - const again = yield* reopened.acquire(request, bind); - expect(again.port).toBe(first.port); - expect((yield* state.read("first"))?.ports).toEqual([ - { key: "db/sql", host: "127.0.0.1", port: first.port }, - ]); - }), - ).pipe(Effect.provide(NodeServices.layer)), -); - -it.live("reports an occupied saved port without moving its assignment", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* state.save({ - id: "stack", - runtime: "native", - identity: { projectRoot: root, branchContext: "test", stackName: "ports" }, - instances: [], - lifetime: "detached", - composition: { members: [], dependencies: [] }, - ports: [], - }); - const ports = yield* makePorts(state); - const scope = yield* Scope.make(); - const request = { stackId: "stack", key: "api", host: "127.0.0.1", port: "auto" as const }; - const first = yield* ports - .acquire(request, bind) - .pipe(Effect.provideService(Scope.Scope, scope)); - yield* Scope.close(scope, Exit.void); - yield* bind("127.0.0.1", first.port); - const failure = yield* ports.acquire(request, bind).pipe(Effect.flip); - expect(failure).toBeInstanceOf(PortError); - expect(failure.message).toContain(`api at 127.0.0.1:${first.port}`); - expect(failure.message).not.toContain("claims this port"); - expect((yield* state.read("stack"))?.ports[0]?.port).toBe(first.port); - }), - ).pipe(Effect.provide(NodeServices.layer)), -); - -it.live("allocates an auto port outside a contiguous range that refuses to bind", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* state.save({ - id: "stack", - runtime: "native", - identity: { projectRoot: root, branchContext: "test", stackName: "ports" }, - instances: [], - lifetime: "detached", - composition: { members: [], dependencies: [] }, - ports: [], - }); - const ports = yield* makePorts(state); - // Reserves most of the span contiguously, as Windows excluded ranges do. - const reservedBelow = 30000; - const acquired = yield* ports.acquire( - { stackId: "stack", key: "api", host: "127.0.0.1", port: "auto" }, - (host, port) => - port < reservedBelow - ? Effect.fail(new PortError({ key: "api", message: `bind EACCES ${host}:${port}` })) - : Effect.succeed(port), - ); - expect(acquired.port).toBeGreaterThanOrEqual(reservedBelow); - expect((yield* state.read("stack"))?.ports).toEqual([ - { key: "api", host: "127.0.0.1", port: acquired.port }, - ]); - }), - ).pipe(Effect.provide(NodeServices.layer)), -); - -it.live("reassigns the same auto port after its claim is released", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* state.save({ - id: "stack", - runtime: "native", - identity: { projectRoot: root, branchContext: "test", stackName: "ports" }, - instances: [], - lifetime: "detached", - composition: { members: [], dependencies: [] }, - ports: [], - }); - const ports = yield* makePorts(state); - const request = { stackId: "stack", key: "api", host: "127.0.0.1", port: "auto" as const }; - const accept = (_host: string, port: number) => Effect.succeed(port); - const first = yield* ports.acquire(request, accept); - yield* ports.release("stack", "api"); - const again = yield* ports.acquire(request, accept); - expect(again.port).toBe(first.port); - }), - ).pipe(Effect.provide(NodeServices.layer)), -); - -it.live("names the last bind failure when no public port is available", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* state.save({ - id: "stack", - runtime: "native", - identity: { projectRoot: root, branchContext: "test", stackName: "ports" }, - instances: [], - lifetime: "detached", - composition: { members: [], dependencies: [] }, - ports: [], - }); - const ports = yield* makePorts(state); - const failure = yield* ports - .acquire({ stackId: "stack", key: "api", host: "127.0.0.1", port: "auto" }, (host, port) => - Effect.fail(new PortError({ key: "api", message: `bind EACCES ${host}:${port}` })), - ) - .pipe(Effect.flip); - expect(failure.message).toContain("No public port is available"); - expect(failure.message).toContain("bind EACCES 127.0.0.1:"); - expect((yield* state.read("stack"))?.ports).toEqual([]); - }), - ).pipe(Effect.provide(NodeServices.layer)), -); - -const saveStack = ( - state: StackNamespace.Interface, - root: string, - id: string, - ports: StackNamespace.SavedStack["ports"] = [], -) => +const saveStack = (state: StackNamespace.Interface, root: string, id: string) => state.save({ id, runtime: "native", - identity: { projectRoot: root, branchContext: `branch-${id}`, stackName: id }, + identity: { projectRoot: root, branchContext: "test", stackName: id }, instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports, + ports: [], }); -it.live("allocates past siblings whose state is unreadable or from a newer format", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* saveStack(state, root, "healthy"); - yield* fs.makeDirectory(`${root}/broken`); - yield* fs.writeFileString(`${root}/broken/state.json`, "{broken"); - yield* fs.makeDirectory(`${root}/newer`); - yield* fs.writeFileString( - `${root}/newer/state.json`, - '{"id":"newer","runtime":"future","ports":[]}', - ); - const ports = yield* makePorts(state); - const acquired = yield* ports.acquire( - { stackId: "healthy", key: "sql", host: "127.0.0.1", port: "auto" }, - bind, - ); - expect((yield* state.read("healthy"))?.ports).toEqual([ - { key: "sql", host: "127.0.0.1", port: acquired.port }, - ]); - expect(yield* fs.readFileString(`${root}/broken/state.json`)).toBe("{broken"); - }), - ).pipe(Effect.provide(NodeServices.layer)), -); +/** Binds like a brand-new listener Network.ts would create: probed first, then really bound, so + * the assertion exercises Linux's real EADDRINUSE path and macOS's probe alike. */ +const probedBind = (key: string) => (host: string, port: number) => + probeVacant(process.platform)(key, host, port).pipe(Effect.andThen(bindTcp(host, port))); -it.live("keeps auto allocation off ports claimed by a sibling in a newer format", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* saveStack(state, root, "stack"); - const ports = yield* makePorts(state); - const request = { stackId: "stack", key: "api", host: "127.0.0.1", port: "auto" as const }; - const accept = (_host: string, port: number) => Effect.succeed(port); - const preferred = yield* ports.acquire(request, accept); - yield* ports.release("stack", "api"); - yield* fs.makeDirectory(`${root}/newer`); - yield* fs.writeFileString( - `${root}/newer/state.json`, - `{"id":"newer","runtime":"future","ports":[{"key":"api","host":"127.0.0.1","port":${preferred.port}}]}`, - ); - const moved = yield* ports.acquire(request, accept); - expect(moved.port).not.toBe(preferred.port); - }), - ).pipe(Effect.provide(NodeServices.layer)), -); - -it.live("lets a stack bind an explicit port that a stopped stack still claims", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* saveStack(state, root, "stopped"); - yield* saveStack(state, root, "current"); - const ports = yield* makePorts(state); - const stoppedScope = yield* Scope.make(); - const stopped = yield* ports - .acquire({ stackId: "stopped", key: "db/sql", host: "127.0.0.1", port: "auto" }, bind) - .pipe(Effect.provideService(Scope.Scope, stoppedScope)); - yield* Scope.close(stoppedScope, Exit.void); +it.live( + "on this platform, auto allocation skips a port a foreign listener occupies, even where overlapping binds would otherwise succeed", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const id = `auto-skip-${randomUUID()}`; + const state = yield* makeTestState(root); + yield* saveStack(state, root, id); + const ports = yield* makePorts(state); + const request = { stackId: id, key: "api", host: "127.0.0.1", port: "auto" as const }; + const bind = probedBind("api"); - const current = yield* ports.acquire( - { stackId: "current", key: "db/sql", host: "127.0.0.1", port: stopped.port }, - bind, - ); - expect(current.port).toBe(stopped.port); - expect((yield* state.read("stopped"))?.ports).toEqual([ - { key: "db/sql", host: "127.0.0.1", port: stopped.port }, - ]); - expect((yield* state.read("current"))?.ports).toEqual([ - { key: "db/sql", host: "127.0.0.1", port: stopped.port }, - ]); - }), - ).pipe(Effect.provide(NodeServices.layer)), -); + const firstScope = yield* Scope.make(); + const first = yield* ports + .acquire(request, bind) + .pipe(Effect.provideService(Scope.Scope, firstScope)); + yield* Scope.close(firstScope, Exit.void); + yield* ports.release(id, "api"); -it.live("names the stack claiming an explicit port that a live listener holds", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* saveStack(state, root, "holder"); - yield* saveStack(state, root, "current"); - const ports = yield* makePorts(state); - const held = yield* ports.acquire( - { stackId: "holder", key: "db/sql", host: "127.0.0.1", port: "auto" }, - bind, - ); + // Reacquiring the same stack and key retries the same hash-seeded candidate first, so + // whatever now occupies it deterministically forces the scan onto the next one. + const foreign = Net.createServer(); + yield* Effect.callback((resume) => { + foreign.once("error", (cause) => resume(Effect.fail(cause))); + foreign.listen(first.port, "127.0.0.1", () => resume(Effect.void)); + }); + yield* Effect.addFinalizer(() => + Effect.callback((resume) => { + foreign.close(() => resume(Effect.void)); + }), + ); - const failure = yield* ports - .acquire({ stackId: "current", key: "db/sql", host: "127.0.0.1", port: held.port }, bind) - .pipe(Effect.flip); - expect(failure).toBeInstanceOf(PortError); - expect(failure.message).toContain(`db/sql at 127.0.0.1:${held.port}`); - expect(failure.message).toContain(`stack "holder" on branch-holder in ${root}`); - expect((yield* state.read("current"))?.ports).toEqual([]); - }), - ).pipe(Effect.provide(NodeServices.layer)), + const second = yield* ports.acquire(request, bind); + expect(second.port).not.toBe(first.port); + }), + ).pipe( + Effect.provide( + Layer.merge( + NodeServices.layer, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), ); -for (const [held, requested] of [ - ["127.0.0.1", "0.0.0.0"], - ["::1", "127.0.0.1"], -] as const) - it.live(`rejects ${requested} on a port a ${held} listener holds where binds can overlap`, () => +it.live( + "a fixed TCP request surfaces a structured, foreign conflict when a real bind fails with EADDRINUSE", + () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped(); + const id = `tcp-foreign-${randomUUID()}`; const state = yield* makeTestState(root); - yield* saveStack(state, root, "holder"); - yield* saveStack(state, root, "current"); - const ports = yield* makePorts(state, "darwin"); - const listener = yield* ports.acquire( - { stackId: "holder", key: "api", host: held, port: "auto" }, - bind, + yield* saveStack(state, root, id); + const ports = yield* makePorts(state); + + const foreign = yield* Effect.acquireRelease( + Effect.callback((resume) => { + const server = Net.createServer(); + server.once("error", (cause) => resume(Effect.fail(cause))); + server.listen(0, "127.0.0.1", () => resume(Effect.succeed(server))); + }), + (server) => + Effect.callback((resume) => { + server.close(() => resume(Effect.void)); + }), ); - const overlappingBinds = yield* Ref.make(0); + const address = foreign.address(); + if (address === null || typeof address === "string") + return yield* Effect.die("Unable to reserve a test port"); + // No probe wrapper here, unlike `Network.ts`'s own callback: this exercises the real TCP + // bind failure (EADDRINUSE) directly, which the probe would otherwise intercept first on a + // platform where overlapping binds succeed. const failure = yield* ports - .acquire({ stackId: "current", key: "api", host: requested, port: listener.port }, () => - Ref.update(overlappingBinds, (count) => count + 1), - ) + .acquire({ stackId: id, key: "api", host: "127.0.0.1", port: address.port }, bindTcp) .pipe(Effect.flip); - expect(failure.message).toContain("already in use"); - expect(failure.message).toContain(`stack "holder"`); - expect(yield* Ref.get(overlappingBinds)).toBe(0); - expect((yield* state.read("current"))?.ports).toEqual([]); + if (!(failure instanceof PortError)) return yield* Effect.die("expected a PortError"); + expect(failure.conflict?.holder).toBe("foreign"); }), - ).pipe(Effect.provide(NodeServices.layer)), - ); + ).pipe( + Effect.provide( + Layer.merge( + NodeServices.layer, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), +); + +it.live( + "interrupting an initial auto acquisition while its bind attempt is pending leaves no row behind", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const id = `auto-interrupt-${randomUUID()}`; + const state = yield* makeTestState(root); + yield* saveStack(state, root, id); + const ports = yield* makePorts(state); + const realRoot = yield* fs.realPath(root); + const portReservations = yield* PortReservations.Service.pipe( + Effect.provide(PortReservations.layer), + ); -it.live("leaves a fixed port to the bind where overlapping binds are rejected", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* saveStack(state, root, "holder"); - yield* saveStack(state, root, "current"); - const ports = yield* makePorts(state, "linux"); - const listener = yield* ports.acquire( - { stackId: "holder", key: "api", host: "127.0.0.1", port: "auto" }, - bind, - ); - const binds = yield* Ref.make(0); + // Stands in for a probe or bind that never settles: the first candidate's row is + // committed, then the acquisition is interrupted while still inside it. + const bind = (_host: string, _port: number) => Effect.never; + const fiber = yield* ports + .acquire({ stackId: id, key: "api", host: "127.0.0.1", port: "auto" }, bind) + .pipe(Effect.forkChild); + const scanned = yield* portReservations.find(realRoot, id, "api").pipe( + Effect.repeat({ + while: (port) => port === undefined, + schedule: Schedule.spaced("5 millis"), + }), + Effect.timeout("5 seconds"), + ); + expect(scanned).toBeDefined(); - const acquired = yield* ports.acquire( - { stackId: "current", key: "api", host: "0.0.0.0", port: listener.port }, - () => Ref.update(binds, (count) => count + 1), - ); - expect(acquired.port).toBe(listener.port); - expect(yield* Ref.get(binds)).toBe(1); - }), - ).pipe(Effect.provide(NodeServices.layer)), + yield* Fiber.interrupt(fiber); + expect(yield* portReservations.find(realRoot, id, "api")).toBeUndefined(); + }), + ).pipe( + Effect.provide( + Layer.merge( + NodeServices.layer, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), ); -it.live("lets a stack bind a port that a running stack claims but does not listen on", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* saveStack(state, root, "running"); - yield* saveStack(state, root, "current"); - const ports = yield* makePorts(state); - yield* ports.acquire( - { stackId: "running", key: "admin", host: "127.0.0.1", port: "auto" }, - bind, - ); - const restScope = yield* Scope.make(); - const rest = yield* ports - .acquire({ stackId: "running", key: "api", host: "127.0.0.1", port: "auto" }, bind) - .pipe(Effect.provideService(Scope.Scope, restScope)); - yield* Scope.close(restScope, Exit.void); +it.live( + "a stack-wide destroy defers a dedicated row's release; a sweep failure keeps it uncertain, and the next destroy, once resolved, releases it", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-defer-destroy-" }); + const id = `defer-destroy-${randomUUID()}`; + const state = yield* makeTestState(`${root}/state`); + const saved = { + id, + runtime: "native" as const, + identity: { projectRoot: root, branchContext: "test", stackName: id }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + + // A leftover claim the sweep must reconcile before destroy can confirm nothing remains. + const claimed = path.join(root, "data", "claimed"); + yield* fs.makeDirectory(claimed, { recursive: true }); + yield* state.claim(id, { kind: "directory", id: claimed }); - const current = yield* ports.acquire( - { stackId: "current", key: "api", host: "127.0.0.1", port: rest.port }, - bind, - ); - expect(current.port).toBe(rest.port); - expect((yield* state.read("current"))?.ports).toEqual([ - { key: "api", host: "127.0.0.1", port: rest.port }, - ]); - }), - ).pipe(Effect.provide(NodeServices.layer)), -); + // Fails removing that one claimed directory on the first attempt only, standing in for any + // concrete sweep failure (an unreachable engine, a locked file, and so on); resolved for the + // retry once the uncertainty would realistically have cleared. + let resolved = false; + const unreliableFileSystem = Layer.effect( + FileSystem.FileSystem, + Effect.map(FileSystem.FileSystem, (real) => + FileSystem.FileSystem.of({ + ...real, + remove: (target, options) => + target === claimed && !resolved + ? Effect.die("Injected sweep failure") + : real.remove(target, options), + }), + ), + ).pipe(Layer.provide(NodeFileSystem.layer)); -it.live("lets exactly one of two stacks sharing a saved port bind it when both start at once", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* saveStack(state, root, "first"); - const ports = yield* makePorts(state); - const request = { stackId: "first", key: "api", host: "127.0.0.1", port: "auto" as const }; - const seedScope = yield* Scope.make(); - const seed = yield* ports - .acquire(request, bind) - .pipe(Effect.provideService(Scope.Scope, seedScope)); - yield* Scope.close(seedScope, Exit.void); - yield* saveStack(state, root, "second", [{ key: "api", host: "127.0.0.1", port: seed.port }]); + const owner = yield* ownerFor({ + saved, + state, + root: `${root}/data`, + cacheRoot: `${root}/cache`, + }).pipe(Effect.provide(unreliableFileSystem)); + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + resolved = true; + yield* owner.namespace.destroy; + }).pipe(Effect.ignore), + ); + // Composing reserves every member's public port up front (including a lazy one like + // mail), with no member ever started: a real reservation with no native artifact needed, + // so this runs on every platform, Windows included. + const definitions = yield* owner.rpc.supabaseComposition({ + services: [{ service: "mail", config: {}, endpoints: { http: { port: "auto" } } }], + }); + const mail = definitions.find((entry) => entry.creation.service === "mail"); + if (mail === undefined) return yield* Effect.die("mail missing from the composition"); - const [first, second] = yield* Effect.all( - [ - Effect.exit(ports.acquire(request, bind)), - Effect.exit(ports.acquire({ ...request, stackId: "second" }, bind)), - ], - { concurrency: "unbounded" }, - ); - const outcomes = [ - { claimant: "second", exit: first }, - { claimant: "first", exit: second }, - ]; - expect(outcomes.filter(({ exit }) => Exit.isSuccess(exit))).toHaveLength(1); - const loser = outcomes.find(({ exit }) => Exit.isFailure(exit)); - const failure = - loser !== undefined && Exit.isFailure(loser.exit) - ? Cause.findErrorOption(loser.exit.cause) - : Option.none(); - if (Option.isNone(failure)) return yield* Effect.die("expected a port conflict"); - expect(failure.value).toBeInstanceOf(PortError); - expect(failure.value.message).toContain(`127.0.0.1:${seed.port}`); - expect(failure.value.message).toContain(`stack "${loser?.claimant}"`); - for (const id of ["first", "second"]) - expect((yield* state.read(id))?.ports).toEqual([ - { key: "api", host: "127.0.0.1", port: seed.port }, - ]); - }), - ).pipe(Effect.provide(NodeServices.layer)), + const realStateRoot = yield* fs.realPath(`${root}/state`); + const portReservations = yield* PortReservations.Service.pipe( + Effect.provide(PortReservations.layer), + ); + expect(yield* portReservations.find(realStateRoot, id, `${mail.id}:http`)).toBeDefined(); + + const firstAttempt = yield* owner.namespace.destroy.pipe(Effect.exit); + expect(firstAttempt._tag).toBe("Failure"); + // Deferred: the row survives a destroy that could not confirm nothing remains. + expect(yield* portReservations.find(realStateRoot, id, `${mail.id}:http`)).toBeDefined(); + + resolved = true; + yield* owner.namespace.destroy; + expect(yield* portReservations.find(realStateRoot, id, `${mail.id}:http`)).toBeUndefined(); + }), + ).pipe( + Effect.provide( + Layer.merge( + NodeServices.layer, + Layer.merge( + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), + ), ); diff --git a/packages/stack/src/Ports.ts b/packages/stack/src/Ports.ts index 82c87f3e15..d38f54dcfb 100644 --- a/packages/stack/src/Ports.ts +++ b/packages/stack/src/Ports.ts @@ -1,6 +1,19 @@ -import { Cause, Data, Effect, Exit, Hash, Option, Scope } from "effect"; +import { + Cause, + Data, + Effect, + Exit, + FileSystem, + Hash, + Option, + Path, + Predicate, + Scope, +} from "effect"; import * as Net from "node:net"; import type * as StackNamespace from "./StackNamespace.ts"; +import * as PortReservations from "./namespace/PortReservations.ts"; +import type { Holder } from "./namespace/PortReservations.ts"; const portBase = 20000; /** Stays below the Linux ephemeral range, per the [architecture ADR](../../../docs/adr/0017-simplified-managed-stack-architecture.md). */ @@ -8,10 +21,18 @@ const portSpan = 12768; /** Co-prime with the span, so the scan visits every port once and steps past reserved ranges. */ const portStride = 257; +/** Why a port could not be used: another stack's live reservation, or a process outside the registry. */ +export interface PortConflict { + readonly port: number; + readonly endpoint: string; + readonly holder: Holder | "foreign"; +} + export class PortError extends Data.TaggedError("PortError")<{ readonly key: string; readonly message: string; readonly cause?: unknown; + readonly conflict?: PortConflict; }> {} export interface PortRequest { @@ -47,177 +68,234 @@ export const accepts = (host: string, port: number) => /** Linux rejects a bind that overlaps a same-family listener on the port; macOS, BSD, and Windows accept it. */ const bindsCanOverlap = (platform: NodeJS.Platform) => platform !== "linux"; -const loopbackOccupied = (port: number) => - Effect.forEach(["127.0.0.1", "::1"], (host) => accepts(host, port), { +const isRefused = (cause: unknown) => + Predicate.hasProperty(cause, "code") && cause.code === "ECONNREFUSED"; + +/** A refused loopback connect can take seconds on Windows; darwin and linux answer almost immediately. */ +const refusalTimeout = (platform: NodeJS.Platform) => + platform === "win32" ? "3 seconds" : "250 millis"; + +/** + * Resolves `true` only on a definite `ECONNREFUSED`, which is the one outcome that proves nothing + * answers `host:port`. An accepted connection, a timeout, or any other error (for example + * `ECONNRESET`) all count as occupied: this is the vacancy probe's own, stricter semantics, kept + * separate from {@link accepts}'s generous "did anything answer" check that `ProcessRecipe` uses + * for readiness. + */ +const refused = (platform: NodeJS.Platform) => (host: string, port: number) => + Effect.callback((resume) => { + const socket = Net.connect({ host: probeHost(host), port }); + const settle = (value: boolean) => { + socket.destroy(); + resume(Effect.succeed(value)); + }; + socket.once("connect", () => settle(false)); + socket.on("error", (cause: unknown) => settle(isRefused(cause))); + return Effect.sync(() => { + socket.destroy(); + }); + }).pipe( + Effect.timeoutOrElse({ + duration: refusalTimeout(platform), + orElse: () => Effect.succeed(false), + }), + ); + +const loopbackVacant = (platform: NodeJS.Platform) => (port: number) => + Effect.forEach(["127.0.0.1", "::1"], (host) => refused(platform)(host, port), { concurrency: "unbounded", - }).pipe(Effect.map((answers) => answers.some(Boolean))); + }).pipe(Effect.map((refusals) => refusals.every(Boolean))); -const claimantOf = ( - stacks: ReadonlyArray, - stackId: string, - port: number, -) => - stacks.find((other) => other.id !== stackId && other.ports.some((claim) => claim.port === port)) - ?.id; +/** The OS error `bindTcp`/`makeHttpProxy` surface when another process already owns the port. */ +const isAddressInUse = (cause: unknown) => + Predicate.hasProperty(cause, "code") && cause.code === "EADDRINUSE"; + +/** + * Guards a brand-new physical listener only; reusing an existing one never probes. On platforms + * where overlapping binds silently succeed, only `ECONNREFUSED` on both loopback families proves + * the port vacant, so anything else (an answer, a timeout, another error) counts as occupied. This + * narrows, but cannot close, the race between the probe and the real bind that follows it: the OS + * still allows a foreign process to bind in between on these platforms. + */ +export const probeVacant = + (platform: NodeJS.Platform) => + (key: string, host: string, port: number): Effect.Effect => + Effect.gen(function* () { + if (!bindsCanOverlap(platform)) return; + if (yield* loopbackVacant(platform)(port)) return; + return yield* new PortError({ + key, + message: `Public port ${port} for ${key} at ${host}:${port} is already in use`, + conflict: { port, endpoint: key, holder: "foreign" }, + }); + }); const resolveRequest = ( stack: StackNamespace.SavedStack, request: PortRequest, + owned: number | undefined, ): Effect.Effect< { readonly saved: StackNamespace.PortClaim | undefined; readonly requested: number | "auto" }, PortError > => { const saved = stack.ports.find((entry) => entry.key === request.key); - if ( - saved !== undefined && - (saved.host !== request.host || (request.port !== "auto" && saved.port !== request.port)) - ) + if (saved !== undefined && saved.host !== request.host) return Effect.fail( new PortError({ key: request.key, message: "The requested listener differs from its saved assignment", }), ); - const requested = saved?.port ?? request.port; - if (requested === "auto") return Effect.succeed({ saved, requested }); - if (!Number.isInteger(requested) || requested < 1 || requested > 65535) - return Effect.fail(new PortError({ key: request.key, message: "Invalid public port" })); - if (stack.ports.some((claim) => claim.key !== request.key && claim.port === requested)) + const expected = owned ?? saved?.port; + if (request.port !== "auto" && expected !== undefined && expected !== request.port) return Effect.fail( new PortError({ key: request.key, - message: `Public port ${requested} is claimed by another listener of this stack`, + message: "The requested listener differs from its reserved assignment", }), ); + const requested = expected ?? request.port; + if (requested === "auto") return Effect.succeed({ saved, requested }); + if (!Number.isInteger(requested) || requested < 1 || requested > 65535) + return Effect.fail(new PortError({ key: request.key, message: "Invalid public port" })); return Effect.succeed({ saved, requested }); }; -/** Claims steer auto allocation away from saved stacks; live listeners and binds decide conflicts for fixed ports. */ -export const makePorts = ( - state: StackNamespace.Interface, - platform: NodeJS.Platform = process.platform, -) => - Effect.sync(() => { - const describe = (id: string) => - state.read(id).pipe( - Effect.map((saved) => - saved === undefined - ? id - : `"${saved.identity.stackName}" on ${saved.identity.branchContext} in ${saved.identity.projectRoot}`, - ), - Effect.orElseSucceed(() => id), - ); +/** + * One stack's public ports, reserved through the per-user registry before any physical listener + * exists: a reservation is committed first, then the listener is created, so a stopped stack's + * ports stay reserved across every other stack's starts and across state roots. + */ +export const makePorts = (state: StackNamespace.Interface) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const portReservations = yield* PortReservations.Service; + // The registry's state_root is the realpath, so a symlinked or relative root still matches + // the identity another process derives from the same stack. + const stateRoot = yield* fs.realPath(state.root); - const claimedBy = ( - stacks: ReadonlyArray, - stackId: string, - port: number, - ) => { - const claimant = claimantOf(stacks, stackId, port); - return claimant === undefined - ? Effect.succeed("") - : describe(claimant).pipe(Effect.map((stack) => `; stack ${stack} claims this port`)); - }; + const describe = (holder: Holder) => `stack ${holder.stackId} at ${holder.stateRoot}`; - // A caller may serve several claims of one stack from a listener it acquired here, so that - // listener is not a foreign occupant. - const held = new Map(); - const hold = (scope: Scope.Scope, stackId: string, port: number) => { - const key = `${stackId}:${port}`; - return Effect.sync(() => held.set(key, (held.get(key) ?? 0) + 1)).pipe( - Effect.andThen( - Scope.addFinalizer( - scope, - Effect.sync(() => { - const remaining = (held.get(key) ?? 1) - 1; - if (remaining > 0) held.set(key, remaining); - else held.delete(key); - }), - ), + /** Only a confirmed `ENOENT` on the holder's own registration makes its reservation stale. */ + const isGone = (holder: Holder) => + fs.stat(path.join(holder.stateRoot, holder.stackId, "state.json")).pipe( + Effect.as(false), + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed(true), ), + Effect.orElseSucceed(() => false), ); - }; - /** Advisory and outside the registry lock; the bind under the lock still decides. */ - const rejectOccupied = Effect.fn("Ports.rejectOccupied")(function* (request: PortRequest) { - const preview = yield* state.read(request.stackId); - if (preview === undefined) return; - const { requested } = yield* resolveRequest(preview, request); - if ( - requested === "auto" || - !bindsCanOverlap(platform) || - held.has(`${request.stackId}:${requested}`) || - !(yield* loopbackOccupied(requested)) - ) - return; - return yield* new PortError({ - key: request.key, - message: `Public port ${requested} for ${request.key} at ${request.host}:${requested} is already in use${yield* claimedBy(yield* state.claims, request.stackId, requested)}`, + /** + * Reserves `port` for `(stackId, key)`, running the lazy-reclamation check against a live + * conflict's own registration before giving up on it. Resolves to the still-live holder on + * conflict, or `undefined` once the row is ours. Untraced: an auto scan calls this per + * candidate port, and the enclosing `Ports.acquire` span already records the attempt count. + */ + const reserveCandidate = Effect.fnUntraced(function* ( + stackId: string, + key: string, + port: number, + ) { + const holder = yield* portReservations.reserve(stateRoot, stackId, key, port); + if (holder === undefined) return undefined; + if (!(yield* isGone(holder))) return holder; + const reclaimed = yield* portReservations.reclaim(port, holder, { + stateRoot, + stackId, + endpoint: key, }); + return reclaimed ? undefined : holder; }); + const holderMessage = (key: string, port: number, stackId: string, holder: Holder) => + holder.stackId === stackId + ? `Public port ${port} for ${key} is claimed by another listener of this stack` + : `Public port ${port} for ${key} is claimed by ${describe(holder)}`; + const acquire = Effect.fn("Ports.acquire")(function* ( request: PortRequest, bind: (host: string, port: number) => Effect.Effect, ) { - yield* rejectOccupied(request); return yield* state.withLock( Effect.gen(function* () { const stack = yield* state.read(request.stackId); if (stack === undefined) return yield* new PortError({ key: request.key, message: "Stack is not registered" }); - const { saved, requested } = yield* resolveRequest(stack, request); - const others = yield* state.claims; - const claimed = new Set([ - ...stack.ports.filter((claim) => claim.key !== request.key).map((claim) => claim.port), - ...others - .filter((other) => other.id !== request.stackId) - .flatMap((other) => other.ports.map((claim) => claim.port)), - ]); - + const owned = yield* portReservations.find(stateRoot, request.stackId, request.key); + const { saved, requested } = yield* resolveRequest(stack, request, owned); const owner = yield* Scope.Scope; - const start = scanStart(stack, request.key); - let failures = 0; - let lastFailure: PortError | undefined; - for (let attempt = 0; attempt < portSpan && failures < 64; attempt++) { - const port = - requested === "auto" - ? portBase + ((start + attempt * portStride) % portSpan) - : requested; - if (requested === "auto" && claimed.has(port)) continue; - const result = yield* Effect.uninterruptibleMask((restore) => + yield* Effect.annotateCurrentSpan({ + "stack.endpoint": request.key, + "stack.port.mode": + requested !== "auto" ? (owned !== undefined ? "owned" : "fixed") : "auto", + }); + + const publishIfNeeded = (port: number) => + saved === undefined || saved.port !== port || saved.host !== request.host + ? state.save({ + ...stack, + ports: [ + ...stack.ports.filter((entry) => entry.key !== request.key), + { key: request.key, host: request.host, port }, + ], + }) + : Effect.void; + + /** Binds `port` and publishes it, against whatever scope the caller provides. */ + const bindAndPublish = (port: number) => + Effect.gen(function* () { + const listener = yield* bind(request.host, port).pipe( + Effect.mapError( + (cause) => + new PortError({ + key: request.key, + message: `Cannot bind ${request.key} at ${request.host}:${port}: ${cause.message}`, + cause: cause.cause, + conflict: + cause.conflict ?? + (isAddressInUse(cause.cause) + ? { port, endpoint: request.key, holder: "foreign" as const } + : undefined), + }), + ), + ); + yield* publishIfNeeded(port); + return { port, listener }; + }); + + /** + * One bind attempt at `port`, in its own forked scope so a failure rolls back both the + * listener and the `state.json` publish together; a success holds the scope open instead. + */ + const attemptBind = (port: number) => + Effect.uninterruptibleMask((restore) => Effect.gen(function* () { const scope = yield* Scope.fork(owner, "sequential"); return yield* restore( - Effect.gen(function* () { - const listener = yield* bind(request.host, port).pipe( - Effect.mapError( - (cause) => - new PortError({ - key: request.key, - message: `Cannot bind ${request.key} at ${request.host}:${port}: ${cause.message}`, - cause, - }), - ), - Effect.provideService(Scope.Scope, scope), - ); - if (saved === undefined) - yield* state.save({ - ...stack, - ports: [...stack.ports, { key: request.key, host: request.host, port }], - }); - return { port, listener }; - }), + bindAndPublish(port).pipe(Effect.provideService(Scope.Scope, scope)), ).pipe( Effect.onExit((exit) => - Exit.isFailure(exit) - ? Scope.close(scope, exit) - : hold(scope, request.stackId, port), + Exit.isFailure(exit) ? Scope.close(scope, exit) : Effect.void, ), Effect.exit, ); }), ); + + if (requested !== "auto") { + if (owned === undefined) { + const holder = yield* reserveCandidate(request.stackId, request.key, requested); + if (holder !== undefined) + return yield* new PortError({ + key: request.key, + message: holderMessage(request.key, requested, request.stackId, holder), + conflict: { port: requested, endpoint: request.key, holder }, + }); + } + const result = yield* attemptBind(requested); if (Exit.isSuccess(result)) return result.value; const error = Cause.findErrorOption(result.cause); if ( @@ -227,15 +305,63 @@ export const makePorts = ( !(error.value instanceof PortError) ) return yield* Effect.failCause(result.cause); - if (requested !== "auto") - return yield* new PortError({ - key: request.key, - message: `${error.value.message}${yield* claimedBy(others, request.stackId, requested)}`, - cause: error.value.cause, - }); + // Owning this port already, or reserving it exactly, is sticky: a bind failure never + // reassigns it, and the reservation survives so the next attempt retries this port. + return yield* error.value; + } + + // A freshly auto-reserved port that fails to bind (including by interruption mid-probe) + // has no restart stickiness to protect, so it must release the row again. Reserving, + // forking the listener's scope, attempting the bind, and rolling both back on failure all + // happen in one uninterruptible mask, with only the bind attempt itself restored to + // interruptible, mirroring `attemptBind` exactly: a pending interrupt delivered during the + // probe surfaces inside that single mask, where rollback can never be skipped. + const attemptCandidate = (port: number) => + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const holder = yield* reserveCandidate(request.stackId, request.key, port); + if (holder !== undefined) return undefined; + const scope = yield* Scope.fork(owner, "sequential"); + return yield* restore( + bindAndPublish(port).pipe(Effect.provideService(Scope.Scope, scope)), + ).pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) + ? Scope.close(scope, exit).pipe( + Effect.andThen( + portReservations.release(stateRoot, request.stackId, request.key), + ), + ) + : Effect.void, + ), + Effect.exit, + ); + }), + ); + + const start = scanStart(stack, request.key); + let failures = 0; + let lastFailure: PortError | undefined; + for (let attempt = 0; attempt < portSpan && failures < 64; attempt++) { + const port = portBase + ((start + attempt * portStride) % portSpan); + const result = yield* attemptCandidate(port); + if (result === undefined) continue; + if (Exit.isSuccess(result)) { + yield* Effect.annotateCurrentSpan({ "stack.port.attempts": attempt + 1 }); + return result.value; + } + const error = Cause.findErrorOption(result.cause); + if ( + Exit.hasInterrupts(result) || + Exit.hasDies(result) || + Option.isNone(error) || + !(error.value instanceof PortError) + ) + return yield* Effect.failCause(result.cause); failures++; lastFailure = error.value; } + yield* Effect.annotateCurrentSpan({ "stack.port.failures": failures }); return yield* new PortError({ key: request.key, message: @@ -251,6 +377,7 @@ export const makePorts = ( const release = Effect.fn("Ports.release")(function* (stackId: string, key: string) { yield* state.withLock( Effect.gen(function* () { + yield* portReservations.release(stateRoot, stackId, key); const stack = yield* state.read(stackId); if (stack !== undefined) yield* state.save({ @@ -261,5 +388,10 @@ export const makePorts = ( ); }); - return { acquire, release }; + /** Releases every reservation a stack holds, regardless of endpoint; used at destroy's success boundary. */ + const releaseStack = Effect.fn("Ports.releaseStack")(function* (stackId: string) { + yield* portReservations.releaseStack(stateRoot, stackId); + }); + + return { acquire, release, releaseStack }; }); diff --git a/packages/stack/src/Proxy.ts b/packages/stack/src/Proxy.ts index 9ac7018056..6003d55cd9 100644 --- a/packages/stack/src/Proxy.ts +++ b/packages/stack/src/Proxy.ts @@ -21,7 +21,14 @@ const proxyError = (cause: unknown) => export const bindTcp = (host: string, port: number) => NodeSocketServer.make({ host, port, allowHalfOpen: true }).pipe( Effect.mapError( - (cause) => new PortError({ key: "tcp", message: "Cannot bind TCP listener", cause }), + (cause) => + // `SocketServerError.reason.cause` is the OS error (for example EADDRINUSE); unwrap it + // here so `Ports.ts`'s conflict detection sees the same shape it gets from `HttpProxy`. + new PortError({ + key: "tcp", + message: "Cannot bind TCP listener", + cause: cause.reason.cause, + }), ), ); diff --git a/packages/stack/src/Rpc.ts b/packages/stack/src/Rpc.ts index acf913c2e9..5696ff1f78 100644 --- a/packages/stack/src/Rpc.ts +++ b/packages/stack/src/Rpc.ts @@ -1,4 +1,4 @@ -import { Exit, Schema } from "effect"; +import { Cause, Exit, Option, Predicate, Schema } from "effect"; import { Rpc, RpcGroup } from "effect/unstable/rpc"; import { ServiceCreation, ServiceCreationInput } from "./services/Catalog.ts"; import { snapshotScopes } from "./services/DatabaseSnapshot.ts"; @@ -6,6 +6,7 @@ import { causeMessage, CompositionConfig, OrchestratorError } from "./Orchestrat import { CommandInvocation } from "./Commands.ts"; import { StackKeysInput } from "./StackNamespace.ts"; import { failureMessage } from "./internal/failure-message.ts"; +import type { PortConflict } from "./Ports.ts"; const Outcome = Schema.Struct({ id: Schema.String, @@ -13,6 +14,16 @@ const Outcome = Schema.Struct({ error: Schema.optionalKey(Schema.String), }); +const ConflictHolder = Schema.Union([ + Schema.Struct({ stackId: Schema.String, stateRoot: Schema.String }), + Schema.Literal("foreign"), +]); +const Conflict = Schema.Struct({ + port: Schema.Int, + endpoint: Schema.String, + holder: ConflictHolder, +}); + /** A typed failure returned by the stack owner. */ export class StackError extends Schema.TaggedError()("StackError", { operation: Schema.String, @@ -26,12 +37,54 @@ export class StackError extends Schema.TaggedError()("StackError", { reason: Schema.optionalKey( Schema.Literals(["owner-unavailable", "release-mismatch", "runtime-unavailable"]), ), + /** The contested public port, when the failure is a port reservation conflict. */ + conflict: Schema.optionalKey(Conflict), }) {} +const isHolder = ( + value: unknown, +): value is { readonly stackId: string; readonly stateRoot: string } => + Predicate.hasProperty(value, "stackId") && + typeof value.stackId === "string" && + Predicate.hasProperty(value, "stateRoot") && + typeof value.stateRoot === "string"; + +const isPortConflict = (value: unknown): value is PortConflict => + Predicate.hasProperty(value, "port") && + typeof value.port === "number" && + Predicate.hasProperty(value, "endpoint") && + typeof value.endpoint === "string" && + Predicate.hasProperty(value, "holder") && + (value.holder === "foreign" || isHolder(value.holder)); + +/** The first `conflict` found by walking a failure's `cause` chain, if any carries one. */ +const findConflict = (cause: unknown, depth = 0): PortConflict | undefined => { + if (depth > 10 || typeof cause !== "object" || cause === null) return undefined; + if ("conflict" in cause && isPortConflict(cause.conflict)) return cause.conflict; + if ("cause" in cause) return findConflict(cause.cause, depth + 1); + return undefined; +}; + +/** A composition failure's member outcomes, each a plain Effect `Exit`, not just this cause's own. */ +const conflictFromOutcomes = ( + outcomes: ReadonlyArray<{ readonly result: Exit.Exit }> | undefined, +): PortConflict | undefined => { + if (outcomes === undefined) return undefined; + for (const { result } of outcomes) { + if (!Exit.isFailure(result)) continue; + const failure = Cause.findErrorOption(result.cause); + if (Option.isNone(failure)) continue; + const conflict = findConflict(failure.value); + if (conflict !== undefined) return conflict; + } + return undefined; +}; + /** Maps an owner failure to the RPC error, preserving per-member composition outcomes. */ export const stackError = (operation: string, cause: unknown): StackError => { if (Schema.is(StackError)(cause)) return cause; - if (cause instanceof OrchestratorError && cause.outcomes !== undefined) + if (cause instanceof OrchestratorError && cause.outcomes !== undefined) { + const conflict = findConflict(cause) ?? conflictFromOutcomes(cause.outcomes); return new StackError({ operation, message: failureMessage(cause), @@ -40,8 +93,15 @@ export const stackError = (operation: string, cause: unknown): StackError => { succeeded: Exit.isSuccess(result), ...(Exit.isFailure(result) ? { error: causeMessage(result.cause) } : {}), })), + ...(conflict === undefined ? {} : { conflict }), }); - return new StackError({ operation, message: failureMessage(cause) }); + } + const conflict = findConflict(cause); + return new StackError({ + operation, + message: failureMessage(cause), + ...(conflict === undefined ? {} : { conflict }), + }); }; const ServiceErrorSchema = Schema.TaggedStruct("ServiceError", { diff --git a/packages/stack/src/StackNamespace.ts b/packages/stack/src/StackNamespace.ts index f750b0cf11..1f6e540019 100644 --- a/packages/stack/src/StackNamespace.ts +++ b/packages/stack/src/StackNamespace.ts @@ -13,7 +13,6 @@ export { StackCredentials, StackKeysInput, StackLifetime, - type StackClaims, } from "./namespace/Registry.ts"; export interface Interface extends Registry.Interface, Lease.Interface, Claims.Interface { diff --git a/packages/stack/src/lifetime.integration.test.ts b/packages/stack/src/lifetime.integration.test.ts index 9998291fbb..120f0ccb17 100644 --- a/packages/stack/src/lifetime.integration.test.ts +++ b/packages/stack/src/lifetime.integration.test.ts @@ -139,9 +139,7 @@ it.live.skipIf(process.platform === "win32")( )(ready); const state = yield* stateFor(stateRoot); expect(yield* state.leased(stackId)).toBe(true); - expect((yield* state.claims).find(({ id }) => id === stackId)?.ports.length).toBeGreaterThan( - 0, - ); + expect((yield* state.read(stackId))?.ports.length).toBeGreaterThan(0); const ownerPid = yield* captureOwnerPid({ stateRoot, cacheRoot }, stackId); const owned = yield* descendantsOf(ownerPid); expect(owned.length, "the owner runs the native mail service").toBeGreaterThan(0); @@ -163,7 +161,7 @@ it.live.skipIf(process.platform === "win32")( yield* released; expect(owned.filter(alive), "native processes die with their owner").toEqual([]); expect(yield* fs.exists(`${stateRoot}/${stackId}`)).toBe(false); - expect((yield* state.claims).some(({ id }) => id === stackId)).toBe(false); + expect(yield* state.read(stackId)).toBeUndefined(); }).pipe(Effect.scoped, Effect.provide(layer)), { timeout: 180_000 }, ); diff --git a/packages/stack/src/namespace/Namespace.integration.test.ts b/packages/stack/src/namespace/Namespace.integration.test.ts index 6a82787c25..ef12c9c6c8 100644 --- a/packages/stack/src/namespace/Namespace.integration.test.ts +++ b/packages/stack/src/namespace/Namespace.integration.test.ts @@ -205,7 +205,7 @@ describe("durable registry", () => { ); it.live.skipIf(process.platform === "win32" || process.getuid?.() === 0)( - "lists and claims readable stacks past a sibling directory it cannot access", + "lists readable stacks past a sibling directory it cannot access", () => run( Effect.gen(function* () { @@ -229,7 +229,6 @@ describe("durable registry", () => { expect((yield* store.list).map(({ id }) => id)).toEqual([initial.id]); expect(reported).toEqual(["locked"]); - expect((yield* store.claims).map(({ id }) => id)).toEqual([initial.id]); }), ), ); diff --git a/packages/stack/src/namespace/PortReservations.ts b/packages/stack/src/namespace/PortReservations.ts new file mode 100644 index 0000000000..f44fafd6bc --- /dev/null +++ b/packages/stack/src/namespace/PortReservations.ts @@ -0,0 +1,380 @@ +import { userInfo } from "node:os"; +import { DatabaseSync } from "node:sqlite"; +import { NodeServices } from "@effect/platform-node"; +import { Context, Effect, FileSystem, Layer, Path, Predicate, Stream } from "effect"; +import { ChildProcess } from "effect/unstable/process"; +import { namespaceError, type NamespaceError } from "./Capabilities.ts"; +import { errcode } from "./drivers/Sqlite.ts"; +import { lstatPath } from "./drivers/FileSystem.ts"; + +/** Identifies the stack that owns a port reservation. */ +export interface Holder { + readonly stackId: string; + readonly stateRoot: string; +} + +const SQLITE_CONSTRAINT_PRIMARYKEY = 1555; + +export interface Interface { + /** The port this stack already reserves for `endpoint`, if it holds one. */ + readonly find: ( + stateRoot: string, + stackId: string, + endpoint: string, + ) => Effect.Effect; + /** + * Reserves `port` for `(stateRoot, stackId, endpoint)`. Resolves to `undefined` once the row is + * committed, or the live {@link Holder} already occupying `port` when it is taken. + */ + readonly reserve: ( + stateRoot: string, + stackId: string, + endpoint: string, + port: number, + ) => Effect.Effect; + /** + * Atomically replaces `port`'s row with `(stateRoot, stackId, endpoint)`, but only while it still + * belongs to `expected`. Resolves to `false` without changing anything once another reservation + * has already claimed, reclaimed, or released it. + */ + readonly reclaim: ( + port: number, + expected: Holder, + reservation: { + readonly stateRoot: string; + readonly stackId: string; + readonly endpoint: string; + }, + ) => Effect.Effect; + /** Releases one endpoint's reservation; a no-op when it holds none. */ + readonly release: ( + stateRoot: string, + stackId: string, + endpoint: string, + ) => Effect.Effect; + /** Releases every reservation a stack holds, at its state root. */ + readonly releaseStack: ( + stateRoot: string, + stackId: string, + ) => Effect.Effect; +} + +export class Service extends Context.Service()( + "@supabase/stack/PortReservations", +) {} + +const schema = ` +CREATE TABLE IF NOT EXISTS reservation ( + port INTEGER PRIMARY KEY, + state_root TEXT NOT NULL, + stack_id TEXT NOT NULL, + endpoint TEXT NOT NULL, + UNIQUE (state_root, stack_id, endpoint) +)`; + +interface HolderRow { + readonly state_root: string; + readonly stack_id: string; +} + +const isPortRow = (value: unknown): value is { readonly port: number } => + Predicate.hasProperty(value, "port") && typeof value.port === "number"; + +const isHolderRow = (value: unknown): value is HolderRow => + Predicate.hasProperty(value, "state_root") && + typeof value.state_root === "string" && + Predicate.hasProperty(value, "stack_id") && + typeof value.stack_id === "string"; + +/** + * One connection per resolved database path for this process's lifetime. POSIX advisory locks (and + * the SQLite locks built on them) belong to a process, not a descriptor: a second connection this + * process opened on the same path would not see the first one's transaction as contention, so every + * caller in this process shares this one connection instead of risking that coalescing. + */ +const connections = new Map(); + +const connectionFor = (file: string): Effect.Effect => + Effect.suspend(() => { + const existing = connections.get(file); + if (existing !== undefined) return Effect.succeed(existing); + return Effect.try({ + try: () => { + const connection = new DatabaseSync(file); + try { + // Short and bounded: real contention between two of this user's processes is brief, and + // a blocking wait here is simpler than an async retry loop for a database this small. Set + // before the schema statement, so even first-time table creation waits out a racing peer + // instead of surfacing a spurious busy error. + connection.exec("PRAGMA busy_timeout = 2000"); + connection.exec(schema); + } catch (error) { + connection.close(); + throw error; + } + connections.set(file, connection); + return connection; + }, + catch: (cause) => namespaceError("open", cause), + }); + }); + +/** + * Confirms `directory` (created 0700 if missing) is a real directory this uid owns, with the + * no-follow {@link lstatPath} so a symlink planted ahead of us is refused rather than traversed. + * Validates before changing any mode, so a symlink is never followed by `chmod` either. + */ +const securePrivateDirectory = Effect.fn("PortReservations.securePrivateDirectory")(function* ( + fs: FileSystem.FileSystem, + directory: string, +) { + yield* fs + .makeDirectory(directory, { recursive: true, mode: 0o700 }) + .pipe(Effect.mapError((cause) => namespaceError("root", cause))); + const info = yield* lstatPath(directory).pipe( + Effect.mapError((cause) => namespaceError("root", cause)), + ); + if (info === undefined || info.type !== "Directory") + return yield* namespaceError("root", `${directory} is not a directory`); + const uid = process.getuid?.(); + if (uid !== undefined && info.uid !== uid) + return yield* namespaceError( + "root", + `${directory} belongs to uid ${String(info.uid)}, not the current user`, + ); + yield* fs.chmod(directory, 0o700).pipe(Effect.mapError((cause) => namespaceError("root", cause))); +}); + +/** `getent passwd ` prints one `name:passwd:uid:gid:gecos:dir:shell` line; field 6 is home. */ +const parseGetent = (output: string): string | undefined => output.trim().split(":")[5]; + +/** `dscacheutil -q user -a uid ` prints `key: value` lines; `dir:` is home. */ +const parseDscacheutil = (output: string): string | undefined => { + for (const line of output.split("\n")) { + const match = /^dir:\s*(.+)$/u.exec(line); + if (match?.[1] !== undefined) return match[1]; + } + return undefined; +}; + +/** + * Resolves the current user's home directly from the OS, never `$HOME`: on POSIX, `os.userInfo()` + * itself prefers `$HOME` once a process starts with it set, which would let a sandboxed or + * overridden environment move the registry. `getent`/`dscacheutil` read the system database + * instead, so a caller's environment cannot steer this. Windows has neither a POSIX uid nor those + * tools, so there `os.userInfo().homedir` reads the user profile directly instead. + */ +const resolvePasswdHome = Effect.fn("PortReservations.resolvePasswdHome")(function* () { + if (process.platform === "win32") { + const home = userInfo().homedir; + if (home.length === 0) + return yield* namespaceError("home", "Unable to resolve the Windows user profile home"); + return home; + } + const uid = process.getuid?.(); + if (uid === undefined) + return yield* namespaceError( + "home", + "No uid to resolve a passwd home for (not a POSIX process)", + ); + const [executable, args, parse] = + process.platform === "darwin" + ? (["dscacheutil", ["-q", "user", "-a", "uid", String(uid)], parseDscacheutil] as const) + : (["getent", ["passwd", String(uid)], parseGetent] as const); + yield* Effect.annotateCurrentSpan({ + "process.executable.name": executable, + "process.arg_count": args.length, + }); + const output = yield* Effect.scoped( + Effect.gen(function* () { + const child = yield* ChildProcess.make(executable, args, { + stdin: "ignore", + stdout: "pipe", + stderr: "ignore", + }); + const [text, exitCode] = yield* Effect.all( + [child.stdout.pipe(Stream.decodeText, Stream.mkString), child.exitCode], + { concurrency: 2 }, + ); + yield* Effect.annotateCurrentSpan("process.exit_code", Number(exitCode)); + return Number(exitCode) === 0 ? text : ""; + }), + ).pipe( + Effect.timeout("5 seconds"), + Effect.mapError((cause) => namespaceError("home", cause)), + ); + const home = parse(output); + if (home === undefined || home.length === 0) + return yield* namespaceError("home", `Unable to resolve the passwd home for uid ${uid}`); + return home; +}); + +/** + * Computed once per process, replayed for every later caller: the OS passwd home for this uid, + * always through the real Node spawner. Tests regularly swap `ChildProcessSpawner` to sandbox + * container orchestration; this low-level OS fact is not part of that contract, and must resolve + * the same way regardless of what the ambient context provides. + */ +const resolvedHome: Effect.Effect = Effect.runSync( + Effect.cached(resolvePasswdHome().pipe(Effect.provide(NodeServices.layer))), +); + +/** + * One SQLite database per OS user at `/.supabase/ports.sqlite`, the only authority on + * which stack owns a public port across every state root on this machine. + */ +const make = Effect.fn("PortReservations.make")(function* (): Effect.fn.Return< + Interface, + NamespaceError, + FileSystem.FileSystem | Path.Path +> { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = path.join(yield* resolvedHome, ".supabase"); + yield* securePrivateDirectory(fs, directory); + const connection = yield* connectionFor(path.join(directory, "ports.sqlite")); + + const toHolder = (row: HolderRow): Holder => ({ + stackId: row.stack_id, + stateRoot: row.state_root, + }); + + const find = Effect.fn("PortReservations.find")(function* ( + stateRoot: string, + stackId: string, + endpoint: string, + ) { + const row = yield* Effect.try({ + try: () => + connection + .prepare( + "SELECT port FROM reservation WHERE state_root = ? AND stack_id = ? AND endpoint = ?", + ) + .get(stateRoot, stackId, endpoint), + catch: (cause) => namespaceError("find", cause), + }); + return isPortRow(row) ? row.port : undefined; + }); + + const holderOf = (port: number): HolderRow | undefined => { + const row = connection + .prepare("SELECT state_root, stack_id FROM reservation WHERE port = ?") + .get(port); + return isHolderRow(row) ? row : undefined; + }; + + // Untraced: `Ports.ts`'s auto scan calls `reserve` and `reclaim` once per candidate port, and its + // own span already records the attempt and failure counts for the whole scan. + const reserve = Effect.fnUntraced(function* ( + stateRoot: string, + stackId: string, + endpoint: string, + port: number, + ) { + return yield* Effect.try({ + try: () => { + const insertRow = () => + connection + .prepare( + "INSERT INTO reservation (port, state_root, stack_id, endpoint) VALUES (?, ?, ?, ?)", + ) + .run(port, stateRoot, stackId, endpoint); + connection.exec("BEGIN IMMEDIATE"); + try { + try { + insertRow(); + } catch (error) { + if (!errcode(error, SQLITE_CONSTRAINT_PRIMARYKEY)) throw error; + // Still holding the write lock taken above: the conflicting row cannot change until we + // commit or roll back, so this read is a consistent view of who really holds `port`. + const row = holderOf(port); + if (row !== undefined) { + connection.exec("ROLLBACK"); + return toHolder(row); + } + // The row that caused the conflict is already gone within this same transaction's own + // view (for example a concurrent release just ahead of us); nothing else can touch it + // while we hold the lock, so retrying the insert now must succeed. + insertRow(); + } + connection.exec("COMMIT"); + return undefined; + } catch (error) { + connection.exec("ROLLBACK"); + throw error; + } + }, + catch: (cause) => namespaceError("reserve", cause), + }); + }); + + const reclaim = Effect.fnUntraced(function* ( + port: number, + expected: Holder, + reservation: { + readonly stateRoot: string; + readonly stackId: string; + readonly endpoint: string; + }, + ) { + return yield* Effect.try({ + try: () => { + connection.exec("BEGIN IMMEDIATE"); + try { + const current = holderOf(port); + if ( + current === undefined || + current.state_root !== expected.stateRoot || + current.stack_id !== expected.stackId + ) { + connection.exec("ROLLBACK"); + return false; + } + connection.prepare("DELETE FROM reservation WHERE port = ?").run(port); + connection + .prepare( + "INSERT INTO reservation (port, state_root, stack_id, endpoint) VALUES (?, ?, ?, ?)", + ) + .run(port, reservation.stateRoot, reservation.stackId, reservation.endpoint); + connection.exec("COMMIT"); + return true; + } catch (error) { + connection.exec("ROLLBACK"); + throw error; + } + }, + catch: (cause) => namespaceError("reclaim", cause), + }); + }); + + const release = Effect.fn("PortReservations.release")(function* ( + stateRoot: string, + stackId: string, + endpoint: string, + ) { + yield* Effect.try({ + try: () => + connection + .prepare("DELETE FROM reservation WHERE state_root = ? AND stack_id = ? AND endpoint = ?") + .run(stateRoot, stackId, endpoint), + catch: (cause) => namespaceError("release", cause), + }); + }); + + const releaseStack = Effect.fn("PortReservations.releaseStack")(function* ( + stateRoot: string, + stackId: string, + ) { + yield* Effect.try({ + try: () => + connection + .prepare("DELETE FROM reservation WHERE state_root = ? AND stack_id = ?") + .run(stateRoot, stackId), + catch: (cause) => namespaceError("release", cause), + }); + }); + + return { find, reserve, reclaim, release, releaseStack }; +}); + +export const layer = Layer.effect(Service, make().pipe(Effect.map(Service.of))); diff --git a/packages/stack/src/namespace/Registry.ts b/packages/stack/src/namespace/Registry.ts index 356d5ea113..4ca573809e 100644 --- a/packages/stack/src/namespace/Registry.ts +++ b/packages/stack/src/namespace/Registry.ts @@ -89,19 +89,10 @@ export const SavedStack = Schema.Struct({ }); export interface SavedStack extends Schema.Schema.Type {} -const ClaimsDocument = Schema.Struct({ ports: Schema.Array(PortClaim) }); - -export interface StackClaims { - readonly id: string; - readonly ports: ReadonlyArray; -} - export interface Interface { readonly read: (id: string) => Effect.Effect; /** Skips each stack entry that stays unreadable after transient retries, reporting it to `onInvalidState`. */ readonly list: Effect.Effect, NamespaceError>; - /** Decodes only each stack's port claims and silently skips entries that cannot provide them. */ - readonly claims: Effect.Effect, NamespaceError>; readonly save: (state: SavedStack) => Effect.Effect; readonly remove: (id: string) => Effect.Effect; /** Not reentrant; wrap metadata updates here, while Ports operations acquire this lock themselves. */ @@ -184,18 +175,6 @@ export const make = ( const list = Effect.fn("Namespace.Registry.list")(() => readEntries(read, (id, error) => options.onInvalidState?.(id, error) ?? Effect.void), ); - const decodeClaims = Schema.decodeEffect(Schema.fromJsonString(ClaimsDocument)); - const readClaims = (id: string) => - fs.readFileString(statePath(id)).pipe( - retryTransientRead, - Effect.flatMap((text) => - decodeClaims(text).pipe(Effect.mapError((cause) => namespaceError("decode", cause))), - ), - Effect.map(({ ports }): StackClaims => ({ id, ports })), - ); - const claims = Effect.fn("Namespace.Registry.claims")(() => - readEntries(readClaims, () => Effect.void), - ); const save = Effect.fn("Namespace.Registry.save")(function* (state: SavedStack) { yield* checkId(state.id); yield* fs @@ -282,5 +261,5 @@ export const make = ( ), ); - return { read, list: list(), claims: claims(), save, remove, withLock }; + return { read, list: list(), save, remove, withLock }; }); diff --git a/packages/stack/src/port-reservation.e2e.test.ts b/packages/stack/src/port-reservation.e2e.test.ts new file mode 100644 index 0000000000..3e7d87dacd --- /dev/null +++ b/packages/stack/src/port-reservation.e2e.test.ts @@ -0,0 +1,465 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { Cause, Effect, Exit, FileSystem, Layer, Option } from "effect"; +import * as Net from "node:net"; +import { create, open, type Observation } from "./effect.ts"; +import { ownerExitProbe, waitForOwnerExit } from "./HostProcess.ts"; +import * as PortReservations from "./namespace/PortReservations.ts"; +import * as StackNamespace from "./StackNamespace.ts"; +import { captureOwnerPid } from "../tests/owner.ts"; +import { destroyTestStack } from "../tests/stack-cleanup.ts"; + +/** + * Step 1.4's per-user port reservation registry: a stopped stack keeps its public ports across + * every other stack's starts and across state roots, a restart recovers the same ports or fails + * with a structured `PortConflict`, and destroy (but never stop or a kill) releases them. + */ + +const servicesLayer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); +const run = (effect: Effect.Effect) => + Effect.scoped(effect).pipe(Effect.provide(servicesLayer)); + +const mailOn = (port: number | "auto") => + ({ service: "mail" as const, config: {}, endpoints: { http: { port } } }) as const; + +const portOf = (observation: Observation): number => { + const endpoint = observation.endpoints.find((candidate) => candidate.name === "http"); + if (endpoint === undefined) throw new Error(`Missing http endpoint in ${observation.id}`); + return endpoint.port; +}; + +/** A plain external listener, not a stack, holding `host:port` outside this package entirely. */ +const foreignListener = (host: string, port: number) => + Effect.acquireRelease( + Effect.callback((resume) => { + const server = Net.createServer(); + server.once("error", (cause) => resume(Effect.fail(cause))); + server.listen(port, host, () => resume(Effect.succeed(server))); + }), + (server) => + Effect.callback((resume) => { + server.close(() => resume(Effect.void)); + }), + ); + +const makeStack = Effect.fn("PortReservation.makeStack")(function* ( + fs: FileSystem.FileSystem, + cacheRoot: string, + runtime: "native" | "docker" = "native", +) { + const root = yield* fs.makeTempDirectoryScoped({ prefix: `port-reservation-${runtime}-` }); + const stack = yield* create({ + projectRoot: root, + stateRoot: `${root}/state`, + cacheRoot, + runtime, + }); + yield* Effect.addFinalizer(() => destroyTestStack(stack)); + return { stack, stateRoot: `${root}/state` }; +}); + +const conflictHolderStackId = (exit: Exit.Exit) => { + if (Exit.isSuccess(exit)) return undefined; + const failure = Cause.findErrorOption(exit.cause); + if (Option.isNone(failure)) return undefined; + const error = failure.value as { readonly conflict?: { readonly holder?: unknown } }; + const holder = error.conflict?.holder; + return typeof holder === "object" && holder !== null && "stackId" in holder + ? (holder as { readonly stackId: string }).stackId + : undefined; +}; + +const conflictIsForeign = (exit: Exit.Exit) => { + if (Exit.isSuccess(exit)) return false; + const failure = Cause.findErrorOption(exit.cause); + if (Option.isNone(failure)) return false; + const error = failure.value as { readonly conflict?: { readonly holder?: unknown } }; + return error.conflict?.holder === "foreign"; +}; + +/** + * Runs `use` with `HOME` pointed at `home`; a spawned owner inherits it at spawn time. Restored + * afterward, so this only ever affects the one owner launched inside `use`. + */ +const withFakeHome = (home: string, use: Effect.Effect) => + Effect.acquireUseRelease( + Effect.sync(() => { + // oxlint-disable-next-line effecttsgo/process-env-in-effect, effecttsgo/process-env -- a one-shot, restored test fixture, not application config. + const previous = process.env.HOME; + // oxlint-disable-next-line effecttsgo/process-env-in-effect, effecttsgo/process-env -- see above. + process.env.HOME = home; + return previous; + }), + () => use, + (previous) => + Effect.sync(() => { + // oxlint-disable-next-line effecttsgo/process-env-in-effect, effecttsgo/process-env -- restores the mutation made above. + if (previous === undefined) delete process.env.HOME; + // oxlint-disable-next-line effecttsgo/process-env-in-effect, effecttsgo/process-env -- restores the mutation made above. + else process.env.HOME = previous; + }), + ); + +it.live( + "a stopped stack's reserved port survives another stack's start in a different state root; a fixed claim on it is a structured PortConflict naming the holder, and the stack restarts on the same port", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ prefix: "port-reservation-cache-" }); + const { stack: stackA, stateRoot: stateRootA } = yield* makeStack(fs, cacheRoot); + const mailA = yield* stackA.services.create(mailOn("auto")); + yield* mailA.start; + yield* mailA.ready; + const port = portOf(yield* mailA.status); + + // A is stopped, not destroyed: its registration, and the public port it was given, persist. + // `Stack.stop` already confirms the owner (and so its listener) has exited. + yield* stackA.stop; + + const { stack: stackB } = yield* makeStack(fs, cacheRoot); + const mailB = yield* stackB.services.create(mailOn(port)); + const startB = yield* Effect.exit(mailB.start); + expect(Exit.isFailure(startB)).toBe(true); + expect(conflictHolderStackId(startB)).toBe(stackA.id); + + // A's restart recovers its own reserved port; B never legitimately held it. + const reopenedA = yield* open({ id: stackA.id, stateRoot: stateRootA, cacheRoot }); + const reopenedMailA = yield* reopenedA.services.get(mailA.id); + yield* reopenedMailA.start; + yield* reopenedMailA.ready; + expect(portOf(yield* reopenedMailA.status)).toBe(port); + + // Destroy releases the reservation: a third stack can now claim the exact same port. + yield* reopenedA.destroy; + const { stack: stackC } = yield* makeStack(fs, cacheRoot); + const mailC = yield* stackC.services.create(mailOn(port)); + yield* mailC.start; + yield* mailC.ready; + expect(portOf(yield* mailC.status)).toBe(port); + }), + ), + { timeout: 120_000 }, +); + +it.live( + "a SIGKILLed owner's reservation survives: another stack's claim on its port is a conflict, the original restarts on the same port, and destroy then releases it", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-kill-cache-", + }); + const { stack: stackA, stateRoot: stateRootA } = yield* makeStack(fs, cacheRoot); + const mailA = yield* stackA.services.create(mailOn("auto")); + yield* mailA.start; + yield* mailA.ready; + const port = portOf(yield* mailA.status); + + const pid = yield* captureOwnerPid({ stateRoot: stateRootA, cacheRoot }, stackA.id); + yield* Effect.sync(() => process.kill(pid, "SIGKILL")); + // The owner's own exit already confirms the kernel has reclaimed its listeners. + yield* waitForOwnerExit(pid, ownerExitProbe(fs)); + + const { stack: stackB } = yield* makeStack(fs, cacheRoot); + const mailB = yield* stackB.services.create(mailOn(port)); + const startB = yield* Effect.exit(mailB.start); + expect(Exit.isFailure(startB)).toBe(true); + expect(conflictHolderStackId(startB)).toBe(stackA.id); + + const reopenedA = yield* open({ id: stackA.id, stateRoot: stateRootA, cacheRoot }); + const reopenedMailA = yield* reopenedA.services.get(mailA.id); + yield* reopenedMailA.start; + yield* reopenedMailA.ready; + expect(portOf(yield* reopenedMailA.status)).toBe(port); + + yield* reopenedA.destroy; + const { stack: stackC } = yield* makeStack(fs, cacheRoot); + const mailC = yield* stackC.services.create(mailOn(port)); + yield* mailC.start; + yield* mailC.ready; + expect(portOf(yield* mailC.status)).toBe(port); + }), + ), + { timeout: 120_000 }, +); + +it.live( + "a foreign listener occupying a stopped stack's saved port fails its restart as a PortConflict(foreign), never reassigning it", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-foreign-cache-", + }); + const { stack: stackA, stateRoot: stateRootA } = yield* makeStack(fs, cacheRoot); + const mailA = yield* stackA.services.create(mailOn("auto")); + yield* mailA.start; + yield* mailA.ready; + const port = portOf(yield* mailA.status); + + yield* stackA.stop; + + const reopenedA = yield* open({ id: stackA.id, stateRoot: stateRootA, cacheRoot }); + const reopenedMailA = yield* reopenedA.services.get(mailA.id); + + // A process outside the registry occupies A's saved port for exactly this inner scope. + yield* Effect.scoped( + Effect.gen(function* () { + yield* foreignListener("127.0.0.1", port); + const restartA = yield* Effect.exit(reopenedMailA.start); + expect(Exit.isFailure(restartA)).toBe(true); + expect(conflictIsForeign(restartA)).toBe(true); + }), + ); + + // Never reassigned: once the foreign listener goes away, the retry recovers the same port. + yield* reopenedMailA.start; + yield* reopenedMailA.ready; + expect(portOf(yield* reopenedMailA.status)).toBe(port); + }), + ), + { timeout: 120_000 }, +); + +it.live( + "deleting a stopped stack's state root makes its reserved ports reclaimable by another stack", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-reclaim-cache-", + }); + // Plain (unscoped): this test deletes `root` itself, so no scope finalizer should also + // try to remove it afterward. + const root = yield* fs.makeTempDirectory({ prefix: "port-reservation-reclaim-a-" }); + const stateRootA = `${root}/state`; + const stackA = yield* create({ + projectRoot: root, + stateRoot: stateRootA, + cacheRoot, + runtime: "native", + }); + const mailA = yield* stackA.services.create(mailOn("auto")); + yield* mailA.start; + yield* mailA.ready; + const port = portOf(yield* mailA.status); + yield* stackA.stop; + + // The whole state root disappears without going through destroy (an unmounted volume, or a + // deleted checkout): a confirmed ENOENT on its registration is what makes its row stale. + yield* fs.remove(root, { recursive: true, force: true }); + + const { stack: stackB } = yield* makeStack(fs, cacheRoot); + const mailB = yield* stackB.services.create(mailOn(port)); + yield* mailB.start; + yield* mailB.ready; + expect(portOf(yield* mailB.status)).toBe(port); + }), + ), + { timeout: 120_000 }, +); + +it.live( + "a native and a Docker stack racing for the same reserved port never both succeed", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-backend-cache-", + }); + // Destroyed, not merely stopped, so the port is genuinely free for B and C to race over; + // a stopped stack would keep the row and both racers would lose to it instead. + const { stack: stackA } = yield* makeStack(fs, cacheRoot, "native"); + const mailA = yield* stackA.services.create(mailOn("auto")); + yield* mailA.start; + yield* mailA.ready; + const port = portOf(yield* mailA.status); + yield* stackA.destroy; + + const { stack: stackB } = yield* makeStack(fs, cacheRoot, "native"); + const { stack: stackC } = yield* makeStack(fs, cacheRoot, "docker"); + const mailB = yield* stackB.services.create(mailOn(port)); + const mailC = yield* stackC.services.create(mailOn(port)); + const [exitB, exitC] = yield* Effect.all( + [Effect.exit(mailB.start), Effect.exit(mailC.start)], + { concurrency: "unbounded" }, + ); + const successes = [exitB, exitC].filter(Exit.isSuccess); + expect(successes).toHaveLength(1); + const loser = Exit.isFailure(exitB) ? exitB : exitC; + expect(conflictHolderStackId(loser)).toBeDefined(); + }), + ), + { timeout: 180_000 }, +); + +it.live( + "an owner spawned with HOME pointed elsewhere still reserves in the real per-user registry", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-home-cache-", + }); + const { stack: stackA } = yield* makeStack(fs, cacheRoot); + const mailA = yield* stackA.services.create(mailOn("auto")); + yield* mailA.start; + yield* mailA.ready; + const port = portOf(yield* mailA.status); + + const fakeHome = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-fake-home-", + }); + // The owner is spawned lazily, on this stack's first RPC call; the fake HOME must still be + // set when that happens, so the whole setup runs inside the same window. + const startB = yield* withFakeHome( + fakeHome, + Effect.gen(function* () { + const { stack: stackB } = yield* makeStack(fs, cacheRoot); + const mailB = yield* stackB.services.create(mailOn(port)); + return yield* Effect.exit(mailB.start); + }), + ); + expect(Exit.isFailure(startB)).toBe(true); + expect(conflictHolderStackId(startB)).toBe(stackA.id); + }), + ), + { timeout: 120_000 }, +); + +it.live( + "a restart republishes state.json when the registry already holds a port its saved claim forgot", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-desync-cache-", + }); + const { stack: stackA, stateRoot: stateRootA } = yield* makeStack(fs, cacheRoot); + const mailA = yield* stackA.services.create(mailOn("auto")); + yield* mailA.start; + yield* mailA.ready; + const port = portOf(yield* mailA.status); + yield* stackA.stop; + + // Simulate a crash between the registry commit and the local `state.json` publish: the + // registry still names this port, but the saved claim that mirrors it is gone. + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: stateRootA })), + ); + const saved = yield* state.read(stackA.id); + if (saved === undefined) return yield* Effect.die("stack state missing"); + yield* state.save({ ...saved, ports: [] }); + + const reopenedA = yield* open({ id: stackA.id, stateRoot: stateRootA, cacheRoot }); + const reopenedMailA = yield* reopenedA.services.get(mailA.id); + yield* reopenedMailA.start; + yield* reopenedMailA.ready; + expect(portOf(yield* reopenedMailA.status)).toBe(port); + expect((yield* state.read(stackA.id))?.ports.some((entry) => entry.port === port)).toBe( + true, + ); + }), + ), + { timeout: 120_000 }, +); + +it.live( + "a restart re-establishes a registry row its saved claim remembers but the registry lost", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-missing-row-cache-", + }); + const { stack: stackA, stateRoot: stateRootA } = yield* makeStack(fs, cacheRoot); + const mailA = yield* stackA.services.create(mailOn("auto")); + yield* mailA.start; + yield* mailA.ready; + const port = portOf(yield* mailA.status); + yield* stackA.stop; + + // Simulate the registry row disappearing independently of `state.json` (for example, a + // registry file replaced out from under a stopped stack): the saved claim still names the + // port, but no row backs it. + const realStateRoot = yield* fs.realPath(stateRootA); + const portReservations = yield* PortReservations.Service.pipe( + Effect.provide(PortReservations.layer), + ); + yield* portReservations.release(realStateRoot, stackA.id, `${mailA.id}:http`); + + const reopenedA = yield* open({ id: stackA.id, stateRoot: stateRootA, cacheRoot }); + const reopenedMailA = yield* reopenedA.services.get(mailA.id); + yield* reopenedMailA.start; + yield* reopenedMailA.ready; + expect(portOf(yield* reopenedMailA.status)).toBe(port); + expect(yield* portReservations.find(realStateRoot, stackA.id, `${mailA.id}:http`)).toBe( + port, + ); + }), + ), + { timeout: 120_000 }, +); + +it.live( + "composing a member on an already-claimed port surfaces a structured conflict naming the holder", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-composition-cache-", + }); + const { stack: stackA } = yield* makeStack(fs, cacheRoot); + const mailA = yield* stackA.services.create(mailOn("auto")); + yield* mailA.start; + yield* mailA.ready; + const port = portOf(yield* mailA.status); + + // The composition registers every member's public port up front, including a lazy + // one like mail, so the conflict surfaces here rather than at a later `.start()`. + const { stack: stackB } = yield* makeStack(fs, cacheRoot); + const composeB = yield* Effect.exit(stackB.composition.supabase([mailOn(port)])); + expect(Exit.isFailure(composeB)).toBe(true); + expect(conflictHolderStackId(composeB)).toBe(stackA.id); + }), + ), + { timeout: 120_000 }, +); + +it.live( + "restarting a composition surfaces a structured conflict from one member's own failing outcome", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-composition-outcome-cache-", + }); + const { stack } = yield* makeStack(fs, cacheRoot); + const members = yield* stack.composition.supabase([mailOn("auto")]); + yield* stack.composition.start; + const statuses = yield* Effect.forEach(members, (member) => member.status); + const mail = statuses.find((status) => status.config.service === "mail"); + if (mail === undefined) return yield* Effect.die("mail missing from the composition"); + const port = portOf(mail); + yield* stack.composition.stop; + + // Composing and starting cleanly the first time means the top-level failure below cannot + // itself carry the conflict; only `startComposition`'s own per-member outcome does. + yield* foreignListener("127.0.0.1", port); + const restart = yield* Effect.exit(stack.composition.start); + expect(Exit.isFailure(restart)).toBe(true); + expect(conflictIsForeign(restart)).toBe(true); + }), + ), + { timeout: 120_000 }, +); diff --git a/packages/stack/tests/whole-stack/scenarios.ts b/packages/stack/tests/whole-stack/scenarios.ts index 7a1cffe511..75c0259744 100644 --- a/packages/stack/tests/whole-stack/scenarios.ts +++ b/packages/stack/tests/whole-stack/scenarios.ts @@ -560,7 +560,9 @@ const resolveRealDockerHost = Effect.gen(function* () { * Proves a running stack writes only inside its state root and artifact cache (plus the project * directory it was handed): points HOME, TMPDIR/TMP/TEMP and the XDG roots at fresh, empty * directories for the stack and its detached owner, then diffs those directories before and after - * a full start/exercise/stop/destroy cycle. + * a full start/exercise/stop/destroy cycle. The per-user port reservation registry resolves its + * home through the OS passwd database (`getent`/`dscacheutil`), not `$HOME`, so it writes outside + * this sandbox entirely and is not expected to appear in either diff. */ export const writeConfinement = (runtime: Runtime) => Effect.scoped( From 63b0112fa02c8c38c65b878ec57b039f3a3026ac Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sun, 4 Oct 2026 06:53:45 +0200 Subject: [PATCH 018/142] fix(stack): publish Vector and Functions configs content-addressed so fresh containers see them --- .../stack/src/functions/FunctionsBootstrap.ts | 81 ++++------- .../functions-bootstrap.integration.test.ts | 57 +++++++- packages/stack/src/internal/content-digest.ts | 13 ++ .../stack/src/internal/generation-publish.ts | 54 +++++++ packages/stack/src/namespace/Paths.ts | 31 ---- .../services/Functions.integration.test.ts | 33 +++-- packages/stack/src/services/Functions.ts | 54 ++++--- .../src/services/Vector.integration.test.ts | 51 +++++++ packages/stack/src/services/Vector.ts | 132 +++++++++--------- 9 files changed, 315 insertions(+), 191 deletions(-) create mode 100644 packages/stack/src/internal/content-digest.ts create mode 100644 packages/stack/src/internal/generation-publish.ts diff --git a/packages/stack/src/functions/FunctionsBootstrap.ts b/packages/stack/src/functions/FunctionsBootstrap.ts index 0615744cc2..749849b234 100644 --- a/packages/stack/src/functions/FunctionsBootstrap.ts +++ b/packages/stack/src/functions/FunctionsBootstrap.ts @@ -1,4 +1,6 @@ -import { Crypto, Data, Effect, FileSystem, Path, PlatformError, Schema } from "effect"; +import { Crypto, Data, Effect, FileSystem, Path, type PlatformError, Schema } from "effect"; +import { contentDigestHex } from "../internal/content-digest.ts"; +import { publishGeneration } from "../internal/generation-publish.ts"; import { StackIdSchema, type StackId } from "../identity/StackId.ts"; export class FunctionsBootstrapError extends Data.TaggedError("FunctionsBootstrapError")<{ @@ -8,6 +10,8 @@ export class FunctionsBootstrapError extends Data.TaggedError("FunctionsBootstra }> {} export interface FunctionsBootstrapOwner { + /** The owned directory a caller mounts statically; generations publish underneath it. */ + readonly root: string; /** Publishes the stack-owned Edge Runtime main service for the current session. */ readonly write: (input: { readonly content: string; @@ -23,6 +27,8 @@ export interface FunctionsBootstrapOwnerOptions { const failure = (message: string, fields: Readonly> = {}) => new FunctionsBootstrapError({ message, ...fields }); +const generationPrefix = "generation-"; + const mapFs = ( path: string, operation: string, @@ -48,61 +54,24 @@ export const makeFunctionsBootstrapOwner = Effect.fn("FunctionsBootstrap.makeOwn }) { if (input.content.includes("\u0000")) return yield* failure("Functions bootstrap contains an invalid character"); - const target = path.join(root, "index.ts"); - const configFile = path.join(root, "deno.json"); - return yield* Effect.gen(function* () { - const token = yield* crypto.randomUUIDv4.pipe( - Effect.mapError((cause) => - failure("Unable to allocate functions bootstrap file", { cause }), - ), - ); - const temporary = path.join(root, `.index.ts.${token}.tmp`); - return yield* Effect.gen(function* () { - yield* mapFs( - root, - "create functions bootstrap directory", - fs.makeDirectory(root, { recursive: true, mode: 0o700 }), - ); - yield* mapFs(root, "secure functions bootstrap directory", fs.chmod(root, 0o700)); - // An empty workspace root stops Deno config discovery before any ancestor package.json or - // workspace; a plain `{}` still joins an ancestor Deno workspace and fails membership. - yield* mapFs( - configFile, - "write functions bootstrap config", - fs.writeFileString(configFile, '{"workspace":[]}\n', { mode: 0o600 }), - ); - yield* Effect.scoped( - Effect.gen(function* () { - const file = yield* mapFs( - temporary, - "create functions bootstrap file", - fs.open(temporary, { flag: "w", mode: 0o600 }), - ); - yield* mapFs( - temporary, - "write functions bootstrap file", - file.writeAll(new TextEncoder().encode(input.content)), - ); - yield* mapFs(temporary, "sync functions bootstrap file", file.sync); - }), - ); - yield* mapFs(temporary, "secure functions bootstrap file", fs.chmod(temporary, 0o600)); - yield* mapFs(target, "publish functions bootstrap file", fs.rename(temporary, target)); - yield* mapFs(target, "secure published functions bootstrap file", fs.chmod(target, 0o600)); - return yield* mapFs( - target, - "resolve published functions bootstrap file", - fs.realPath(target), - ); - }).pipe( - Effect.ensuring( - fs - .remove(temporary, { force: true }) - .pipe(Effect.catchTag("PlatformError", () => Effect.void)), - ), - ); - }); + const hash = yield* mapFs( + root, + "hash functions bootstrap content", + contentDigestHex(crypto, input.content), + ); + const generation = yield* mapFs( + root, + "publish functions bootstrap generation", + // An empty workspace root stops Deno config discovery before any ancestor package.json or + // workspace; a plain `{}` still joins an ancestor Deno workspace and fails membership. + publishGeneration(fs, path, root, `${generationPrefix}${hash}`, [ + { name: "deno.json", content: '{"workspace":[]}\n', mode: 0o600 }, + { name: "index.ts", content: input.content, mode: 0o600 }, + ]), + ); + const target = path.join(generation, "index.ts"); + return yield* mapFs(target, "resolve published functions bootstrap file", fs.realPath(target)); }); - return { write }; + return { root, write }; }); diff --git a/packages/stack/src/functions/functions-bootstrap.integration.test.ts b/packages/stack/src/functions/functions-bootstrap.integration.test.ts index fd0c43b084..257420668e 100644 --- a/packages/stack/src/functions/functions-bootstrap.integration.test.ts +++ b/packages/stack/src/functions/functions-bootstrap.integration.test.ts @@ -1,6 +1,6 @@ import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; -import { Effect, FileSystem, Path } from "effect"; +import { Deferred, Effect, FileSystem, Fiber, Path } from "effect"; import { StackIdSchema } from "../identity/StackId.ts"; import { makeFunctionsBootstrapOwner } from "./FunctionsBootstrap.ts"; @@ -17,15 +17,26 @@ const setupBootstrapOwner = (prefix: string) => return { fs, root, stackId, owner }; }); +/** `ino` for every file a generation publishes together, keyed by name. */ +const generationInodes = (fs: FileSystem.FileSystem, path: Path.Path, target: string) => + Effect.gen(function* () { + const generation = path.dirname(target); + return { + index: (yield* fs.stat(path.join(generation, "index.ts"))).ino, + config: (yield* fs.stat(path.join(generation, "deno.json"))).ino, + }; + }); + describe("functions bootstrap owner", () => { it.live("publishes a private Functions bootstrap file with restrictive modes", () => Effect.gen(function* () { const path = yield* Path.Path; const { fs, root, owner } = yield* setupBootstrapOwner("stack-functions-bootstrap-"); const target = yield* owner.write({ content: "export default 1" }); - expect(target).toContain( - path.join(root, "functions-one", "runtime", "functions", "index.ts"), + expect(path.dirname(path.dirname(target))).toContain( + path.join(root, "functions-one", "runtime", "functions"), ); + expect(path.basename(target)).toBe("index.ts"); expect(((yield* fs.stat(path.dirname(target))).mode ?? 0) & 0o777).toBe(0o700); expect(((yield* fs.stat(target)).mode ?? 0) & 0o777).toBe(0o600); expect(yield* fs.readFileString(target)).toBe("export default 1"); @@ -56,11 +67,47 @@ describe("functions bootstrap owner", () => { "functions-one", "runtime", "functions", - "index.ts", ); - expect(target).toBe(expected); + expect(path.dirname(path.dirname(target))).toBe(expected); + expect(path.basename(target)).toBe("index.ts"); expect(yield* fs.readFileString(target)).toBe("export default 2"); expect(((yield* fs.stat(target)).mode ?? 0) & 0o777).toBe(0o600); }).pipe(Effect.provide(NodeServices.layer)), ); + + it.live("keeps every published file's identity when identical content is written twice", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const { fs, owner } = yield* setupBootstrapOwner("stack-functions-bootstrap-idempotent-"); + const first = yield* owner.write({ content: "export default 1" }); + const before = yield* generationInodes(fs, path, first); + + const second = yield* owner.write({ content: "export default 1" }); + + expect(second).toBe(first); + expect(yield* generationInodes(fs, path, second)).toEqual(before); + expect(yield* fs.readFileString(first)).toBe("export default 1"); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("converges concurrent writers of identical content on one complete generation", () => + Effect.gen(function* () { + const { fs, owner } = yield* setupBootstrapOwner("stack-functions-bootstrap-race-"); + const gate = yield* Deferred.make(); + const fibers = yield* Effect.forEach( + Array.from({ length: 20 }, () => undefined), + () => + Deferred.await(gate).pipe( + Effect.andThen(owner.write({ content: "export default race" })), + Effect.forkScoped, + ), + ); + yield* Deferred.succeed(gate, undefined); + const targets = yield* Effect.forEach(fibers, (fiber) => Fiber.join(fiber)); + + expect(new Set(targets).size).toBe(1); + const target = targets[0]!; + expect(yield* fs.readFileString(target)).toBe("export default race"); + }).pipe(Effect.provide(NodeServices.layer)), + ); }); diff --git a/packages/stack/src/internal/content-digest.ts b/packages/stack/src/internal/content-digest.ts new file mode 100644 index 0000000000..5be27d5e65 --- /dev/null +++ b/packages/stack/src/internal/content-digest.ts @@ -0,0 +1,13 @@ +import { Effect, type Crypto, type PlatformError } from "effect"; + +const hex = (bytes: Uint8Array): string => + Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""); + +/** A 16-character hex SHA-256 prefix, used to name content-addressed files and directories. */ +export const contentDigestHex = ( + crypto: Crypto.Crypto, + content: string, +): Effect.Effect => + crypto + .digest("SHA-256", new TextEncoder().encode(content)) + .pipe(Effect.map((digest) => hex(digest).slice(0, 16))); diff --git a/packages/stack/src/internal/generation-publish.ts b/packages/stack/src/internal/generation-publish.ts new file mode 100644 index 0000000000..4d0b4c7f4d --- /dev/null +++ b/packages/stack/src/internal/generation-publish.ts @@ -0,0 +1,54 @@ +import { Effect, type FileSystem, type Path, type PlatformError } from "effect"; + +export interface GenerationFile { + readonly name: string; + readonly content: string; + readonly mode: number; +} + +let counter = 0; + +/** + * Publishes `files` together as `root/name` by renaming one staged directory into place, so readers + * never see a partial generation. A rename can't replace a non-empty directory, so a collision with + * an already-complete generation (same content-derived name) is success. + */ +export const publishGeneration = Effect.fn("Internal.publishGeneration")(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + root: string, + name: string, + files: ReadonlyArray, +) { + const generation = path.join(root, name); + const isComplete = (directory: string): Effect.Effect => + Effect.forEach(files, (file) => fs.exists(path.join(directory, file.name))).pipe( + Effect.map((results) => results.every(Boolean)), + ); + if (yield* isComplete(generation)) return generation; + yield* fs.makeDirectory(root, { recursive: true, mode: 0o700 }); + const stage = path.join(root, `.${name}.${process.pid}-${++counter}.tmp`); + // Uninterruptible: an abandoned in-flight rename could otherwise publish a stage that cleanup has + // already partly removed. + yield* Effect.gen(function* () { + yield* fs.makeDirectory(stage, { recursive: true, mode: 0o700 }); + for (const file of files) + yield* fs.writeFileString(path.join(stage, file.name), file.content, { mode: file.mode }); + yield* fs.rename(stage, generation).pipe( + Effect.catch((cause) => + isComplete(generation).pipe( + Effect.orElseSucceed(() => false), + Effect.flatMap((complete) => (complete ? Effect.void : Effect.fail(cause))), + ), + ), + ); + }).pipe( + Effect.ensuring( + fs + .remove(stage, { recursive: true, force: true }) + .pipe(Effect.catchTag("PlatformError", () => Effect.void)), + ), + Effect.uninterruptible, + ); + return generation; +}); diff --git a/packages/stack/src/namespace/Paths.ts b/packages/stack/src/namespace/Paths.ts index 2ca179f325..a6f505f91c 100644 --- a/packages/stack/src/namespace/Paths.ts +++ b/packages/stack/src/namespace/Paths.ts @@ -163,34 +163,3 @@ export const destroyOwnedRoot = Effect.fn("Namespace.Paths.destroyOwnedRoot")( .pipe(Effect.mapError((cause) => onError("destroy", cause))); }), ); - -/** - * Writes `content` to `target` atomically (full content staged beside it, then renamed into - * place), for a file the namespace owns inside a directory a caller may also write into. - */ -export const writeOwnedFile = Effect.fn("Namespace.Paths.writeOwnedFile")( - ( - fs: FileSystem.FileSystem, - path: Path.Path, - target: string, - content: string, - onError: (operation: string, cause: unknown) => E, - ): Effect.Effect => - Effect.gen(function* () { - const directory = path.dirname(target); - yield* fs.makeDirectory(directory, { recursive: true, mode: 0o700 }); - yield* Effect.acquireUseRelease( - fs.makeTempDirectory({ directory, prefix: `.${path.basename(target)}-` }), - (temporaryDirectory) => - Effect.gen(function* () { - const temporary = path.join(temporaryDirectory, path.basename(target)); - yield* fs.writeFileString(temporary, content, { mode: 0o644 }); - yield* fs.rename(temporary, target); - }), - (temporaryDirectory) => - fs - .remove(temporaryDirectory, { recursive: true, force: true }) - .pipe(Effect.catchTag("PlatformError", () => Effect.void)), - ); - }).pipe(Effect.mapError((cause) => onError("prepare", cause))), -); diff --git a/packages/stack/src/services/Functions.integration.test.ts b/packages/stack/src/services/Functions.integration.test.ts index ccc785646c..c1462c2321 100644 --- a/packages/stack/src/services/Functions.integration.test.ts +++ b/packages/stack/src/services/Functions.integration.test.ts @@ -16,10 +16,12 @@ import { } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import type { FunctionsBootstrapOwner } from "../functions/FunctionsBootstrap.ts"; import { ContainerError, type ContainerRuntime } from "../runtime/Container.ts"; import { makeService } from "../Service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import * as Functions from "./Functions.ts"; +import { makeProcessRecipe } from "./ProcessRecipe.ts"; import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; @@ -626,7 +628,23 @@ it.effect("passes POSIX project paths to a docker Functions container from a Win }, }, }; - const recipe = yield* Functions.makeRecipe( + // A real bootstrap owner would join the Win32 stack root below with real fs calls, writing a + // garbled path on this POSIX test host; a fake owner keeps the test to what it exercises + // here, the Windows caller-path translation in `env`/`args`. + const fakeBootstrap: FunctionsBootstrapOwner = { + root: "/fake-bootstrap-root", + write: () => Effect.succeed("/fake-bootstrap-root/generation-fake/index.ts"), + }; + const deps = { + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), + fs: yield* FileSystem.FileSystem, + path: yield* Path.Path.pipe(Effect.provide(NodePath.layerWin32)), + crypto: yield* Crypto.Crypto, + client: yield* HttpClient.HttpClient, + spawner: yield* ChildProcessSpawner.ChildProcessSpawner, + container, + }; + const recipe = yield* makeProcessRecipe( creation, { containerClaims: noContainerClaims, @@ -637,15 +655,8 @@ it.effect("passes POSIX project paths to a docker Functions container from a Win cacheRoot: "C:\\Users\\dev\\cache", runtime: "docker", }, - { - borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), - fs: yield* FileSystem.FileSystem, - path: yield* Path.Path.pipe(Effect.provide(NodePath.layerWin32)), - crypto: yield* Crypto.Crypto, - client: yield* HttpClient.HttpClient, - spawner: yield* ChildProcessSpawner.ChildProcessSpawner, - container, - }, + deps, + Functions.makeSpec(fakeBootstrap, deps.path, deps.fs), ); const scope = yield* Scope.make(); yield* recipe.definition @@ -657,7 +668,7 @@ it.effect("passes POSIX project paths to a docker Functions container from a Win "/__supabase_project/supabase/functions", ); expect(spec?.env.SUPABASE_INTERNAL_FUNCTIONS_FILES_ROOT).toBe("/__supabase_project"); - expect(spec?.args).toContain("--main-service=/__supabase_functions"); + expect(spec?.args).toContain("--main-service=/__supabase_bootstrap/generation-fake"); const config = yield* Schema.decodeUnknownEffect( Schema.fromJsonString( Schema.Record( diff --git a/packages/stack/src/services/Functions.ts b/packages/stack/src/services/Functions.ts index 403dcdc6de..c03bdec3b2 100644 --- a/packages/stack/src/services/Functions.ts +++ b/packages/stack/src/services/Functions.ts @@ -1,4 +1,4 @@ -import { Crypto, Effect, FileSystem, Path, Ref, Schema } from "effect"; +import { Crypto, Effect, FileSystem, Path, Schema } from "effect"; import { makeFunctionsBootstrapOwner, type FunctionsBootstrapOwner, @@ -63,8 +63,10 @@ const FunctionsRuntimeConfigJson = Schema.fromJsonString( Schema.Record(Schema.String, FunctionSettings), ); -const makeSpec = ( - functionsRoot: Ref.Ref, +/** Fixed container mount point for `bootstrap.root`; `args` picks the generation underneath it. */ +const containerBootstrapRoot = "/__supabase_bootstrap"; + +export const makeSpec = ( bootstrap: FunctionsBootstrapOwner, path: Path.Path, fs: FileSystem.FileSystem, @@ -174,12 +176,21 @@ const makeSpec = ( }), args: (creation, endpoints, context) => Effect.gen(function* () { - const override = yield* Ref.get(functionsRoot); + const published = yield* bootstrap + .write({ content: creation.config.bootstrap ?? defaultFunctionsBootstrap }) + .pipe( + Effect.mapError( + (cause) => + new ServiceError({ + operation: "launch", + message: "Unable to publish Functions bootstrap", + cause, + }), + ), + ); const root = context.container - ? override === undefined - ? "/__supabase_functions" - : "/__supabase_bootstrap" - : (override ?? creation.config.functionsRoot); + ? `${containerBootstrapRoot}/${path.basename(path.dirname(published))}` + : path.dirname(published); const http = endpoints.get("http"); const inspector = endpoints.get("inspector"); return [ @@ -193,22 +204,20 @@ const makeSpec = ( ]; }), mounts: (creation) => - Effect.gen(function* () { - const override = yield* Ref.get(functionsRoot); - const files = creation.config.filesRoot; - const source = files ?? creation.config.functionsRoot; - const target = files === undefined ? "/__supabase_functions" : "/__supabase_project"; - return [ - { source, target, readOnly: true }, - ...(override === undefined - ? [] - : [{ source: override, target: "/__supabase_bootstrap", readOnly: true }]), - ]; - }), + Effect.succeed([ + { + source: creation.config.filesRoot ?? creation.config.functionsRoot, + target: + creation.config.filesRoot === undefined ? "/__supabase_functions" : "/__supabase_project", + readOnly: true, + }, + { source: bootstrap.root, target: containerBootstrapRoot, readOnly: true }, + ]), startupCommands: [], + // Prefetches the bootstrap so a later start need not pay for it; `args` publishes idempotently. prepare: (creation) => bootstrap.write({ content: creation.config.bootstrap ?? defaultFunctionsBootstrap }).pipe( - Effect.flatMap((target) => Ref.set(functionsRoot, path.dirname(target))), + Effect.asVoid, Effect.mapError( (cause) => new ServiceError({ @@ -235,7 +244,6 @@ export const makeRecipe = Effect.fn("Functions.makeRecipe")( deps: ProcessDependencies, ): Effect.Effect, CatalogError> => Effect.gen(function* () { - const functionsRoot = yield* Ref.make(undefined); const stackId = yield* Schema.decodeEffect(StackIdSchema)(options.stackId).pipe( Effect.mapError( (cause) => @@ -269,7 +277,7 @@ export const makeRecipe = Effect.fn("Functions.makeRecipe")( creation, options, deps, - makeSpec(functionsRoot, bootstrap, deps.path, deps.fs), + makeSpec(bootstrap, deps.path, deps.fs), ); }), ); diff --git a/packages/stack/src/services/Vector.integration.test.ts b/packages/stack/src/services/Vector.integration.test.ts index 4250a8b401..32412cac2a 100644 --- a/packages/stack/src/services/Vector.integration.test.ts +++ b/packages/stack/src/services/Vector.integration.test.ts @@ -98,6 +98,57 @@ describe("vector recipe", () => { { timeout: 120_000 }, ); + it.live( + "keeps the published config files' identity across a restart with unchanged content", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-vector-restart-" }); + const configRoot = `${root}/vector/runtime/vector`; + const recipe = yield* makeServiceRecipe( + { + service: "vector", + config: { analyticsUrl: "http://analytics" }, + endpoints: { http: { port: "auto" } }, + }, + options(root, "docker"), + ); + const vector = yield* makeService(recipe.definition, { + id: "vector", + config: recipe.creation, + }); + const inodesByFile = Effect.gen(function* () { + const [generation, ...rest] = (yield* fs.readDirectory(configRoot)).filter((entry) => + entry.startsWith("generation-"), + ); + expect(rest).toEqual([]); + const stats = yield* Effect.forEach( + ["vector-api.yaml", "vector-pipeline.yaml"], + (name) => fs.stat(`${configRoot}/${generation}/${name}`), + ); + return { generation, inodes: stats.map((stat) => stat.ino) }; + }); + + yield* vector.start; + yield* vector.ready; + const before = yield* inodesByFile; + + // A docker Vector launch always binds `0.0.0.0:9001` inside the container regardless of + // the published host port, so a restart with the same config republishes byte-identical + // content: the generation name, and every file's identity, must not change. + yield* vector.restart(); + yield* vector.ready; + const after = yield* inodesByFile; + + expect(after.generation).toBe(before.generation); + expect(after.inodes).toEqual(before.inodes); + yield* vector.destroy; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 120_000 }, + ); + it.live("rejects a caller pipeline that resolves to a stack-owned config file", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/services/Vector.ts b/packages/stack/src/services/Vector.ts index 2c14d71443..396435d52a 100644 --- a/packages/stack/src/services/Vector.ts +++ b/packages/stack/src/services/Vector.ts @@ -1,6 +1,7 @@ -import { Effect, type FileSystem, type Path, Schema } from "effect"; +import { type Crypto, Effect, type FileSystem, type Path, Schema } from "effect"; +import { contentDigestHex } from "../internal/content-digest.ts"; +import { publishGeneration } from "../internal/generation-publish.ts"; import { ServiceError } from "../Service.ts"; -import { writeOwnedFile } from "../namespace/Paths.ts"; import { type CatalogOptions, EndpointIntent, serviceCreation } from "./Recipe.ts"; import { requiredInput } from "./ServiceConfig.ts"; import { @@ -57,40 +58,19 @@ const defaultPipelineConfig = [ "", ].join("\n"); -const containerPipelinePath = "/etc/vector/vector.yaml"; -const containerApiPath = "/etc/supabase/vector-api.yaml"; - -const writeAtomically = ( - fs: FileSystem.FileSystem, - path: Path.Path, - target: string, - content: string, -) => - writeOwnedFile( - fs, - path, - target, - content, - (operation, cause) => - new ServiceError({ - operation, - message: "Unable to write Vector config", - cause, - }), - ); +/** Fixed mount point for `configRoot`; `args` names the generation to load underneath it. */ +const containerConfigDir = "/etc/supabase/vector"; +const apiFileName = "vector-api.yaml"; +const pipelineFileName = "vector-pipeline.yaml"; +const generationPrefix = "generation-"; const makeSpec = ( instanceRoot: string, fs: FileSystem.FileSystem, path: Path.Path, + crypto: Crypto.Crypto, ): ProcessRecipeSpec => { const configRoot = path.join(instanceRoot, "runtime", "vector"); - const apiConfigPath = path.join(configRoot, "vector-api.yaml"); - const defaultPipelinePath = path.join(configRoot, "vector.yaml"); - const renderedPipelinePath = path.join(configRoot, "vector.rendered.yaml"); - /** What Vector actually loads: a caller's file is rendered to a recipe-owned copy first. */ - const resolvedPipelinePath = (creation: Creation) => - creation.config.configPath === undefined ? defaultPipelinePath : renderedPipelinePath; return { service: "vector", executable: "bin/vector", @@ -112,44 +92,66 @@ const makeSpec = ( http === undefined ? undefined : `${context.container ? "0.0.0.0" : "127.0.0.1"}:${http.port}`; - yield* writeAtomically(fs, path, apiConfigPath, apiConfigFor(address)); - if (creation.config.configPath !== undefined) { - // Validated by `callerPaths` before prepare ran: read-only, no delete authority over it. - const source = yield* fs.readFileString(creation.config.configPath).pipe( - Effect.mapError( - (cause) => - new ServiceError({ - operation: "prepare", - message: "Unable to read Vector configPath", - cause, - }), - ), - ); - const rendered = renderKnownPlaceholders(source, { - LOGFLARE_URL: creation.config.analyticsUrl, - LOGFLARE_PRIVATE_ACCESS_TOKEN: creation.config.apiKey, - }); - yield* writeAtomically(fs, path, renderedPipelinePath, rendered); - } + const apiContent = apiConfigFor(address); + const callerPath = creation.config.configPath; + const pipelineContent = + callerPath === undefined + ? defaultPipelineConfig + : yield* fs.readFileString(callerPath).pipe( + Effect.map((source) => + renderKnownPlaceholders(source, { + LOGFLARE_URL: creation.config.analyticsUrl, + LOGFLARE_PRIVATE_ACCESS_TOKEN: creation.config.apiKey, + }), + ), + Effect.mapError( + (cause) => + new ServiceError({ + operation: "launch", + message: "Unable to read Vector configPath", + cause, + }), + ), + ); + // Both files publish together under one generation name, so a fresh container only ever + // bind-mounts a directory that has never been deleted before. Length-prefixing the first + // part avoids any ambiguity from concatenating the two strings directly. + const generation = yield* contentDigestHex( + crypto, + `${apiContent.length}:${apiContent}${pipelineContent}`, + ).pipe( + Effect.flatMap((hash) => + publishGeneration(fs, path, configRoot, `${generationPrefix}${hash}`, [ + { name: apiFileName, content: apiContent, mode: 0o644 }, + { name: pipelineFileName, content: pipelineContent, mode: 0o644 }, + ]), + ), + Effect.mapError( + (cause) => + new ServiceError({ + operation: "launch", + message: "Unable to write Vector config", + cause, + }), + ), + ); return context.container - ? ["--config", containerPipelinePath, "--config", containerApiPath] - : ["--config", resolvedPipelinePath(creation), "--config", apiConfigPath]; + ? [ + "--config", + `${containerConfigDir}/${path.basename(generation)}/${pipelineFileName}`, + "--config", + `${containerConfigDir}/${path.basename(generation)}/${apiFileName}`, + ] + : [ + "--config", + path.join(generation, pipelineFileName), + "--config", + path.join(generation, apiFileName), + ]; }), - mounts: (creation) => - Effect.succeed([ - { source: resolvedPipelinePath(creation), target: containerPipelinePath, readOnly: true }, - { source: apiConfigPath, target: containerApiPath, readOnly: true }, - ]), + mounts: () => + Effect.succeed([{ source: configRoot, target: containerConfigDir, readOnly: true }]), startupCommands: [], - prepare: (creation) => - Effect.gen(function* () { - const callerPath = creation.config.configPath; - // A safe placeholder until `args` writes the real address — the listening port isn't - // known until endpoints are reserved for an actual launch, which happens after `prepare`. - yield* writeAtomically(fs, path, apiConfigPath, apiConfigFor(undefined)); - if (callerPath === undefined) - yield* writeAtomically(fs, path, defaultPipelinePath, defaultPipelineConfig); - }), // No removeData: configRoot lives under instanceRoot, which ProcessRecipe's destroyOwnedRoot // already removes recursively; a caller's borrowed config sits outside it by construction, // validated by callerPaths against the whole stack data root, not just this instance root. @@ -164,6 +166,6 @@ export const makeRecipe = Effect.fn("Vector.makeRecipe")( creation, options, deps, - makeSpec(deps.path.join(options.root, options.instanceId), deps.fs, deps.path), + makeSpec(deps.path.join(options.root, options.instanceId), deps.fs, deps.path, deps.crypto), ), ); From f39e96af9a21769979f35b4cb5e956ac8669de4e Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sun, 4 Oct 2026 08:43:10 +0200 Subject: [PATCH 019/142] chore(stack): enforce namespace import boundaries with lint --- .oxlintrc.json | 383 ++++++++++++++++++ .../stack/src/namespace/PortReservations.ts | 41 +- .../stack/src/namespace/drivers/Sqlite.ts | 33 ++ 3 files changed, 425 insertions(+), 32 deletions(-) diff --git a/.oxlintrc.json b/.oxlintrc.json index 8c10b88c04..10f646e45d 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -43,6 +43,389 @@ "rules": { "typescript/await-thenable": "off" } + }, + { + // The namespace owns claims, paths, leases, artifact pins, and port reservations; everywhere + // in the repo, outside `src/namespace/**` itself, reaches that state through + // `StackNamespace.ts`, not by importing namespace internals directly. Repo-wide (apps/cli + // included) so a deep or relative import of a namespace module fails from any package; the + // stack package's own raw-primitive restrictions are layered on top, further down. + "files": ["**/*"], + "excludeFiles": ["packages/stack/src/namespace/**", "packages/stack/src/StackNamespace.ts"], + "rules": { + "no-restricted-imports": [ + "error", + { + "patterns": [ + { + "group": ["**/namespace/**"], + "message": "Namespace internals are private; consume them through StackNamespace.ts." + } + ] + } + ] + } + }, + { + // Default for the whole stack package (`src/**` and `tests/**`): on top of the repo-wide + // namespace restriction above, raw `node:fs`/`node:sqlite` are confined to namespace drivers + // and a short enumerated list of existing files. The files below are pre-existing direct + // namespace consumers the facade doesn't cover yet (tracked for step 3) or raw-primitive + // users; they get their own, narrower overrides further down. + "files": ["packages/stack/src/**", "packages/stack/tests/**"], + "excludeFiles": [ + "packages/stack/src/namespace/**", + "packages/stack/src/StackNamespace.ts", + "packages/stack/src/Network.ts", + "packages/stack/src/Owner.ts", + "packages/stack/src/Ports.ts", + "packages/stack/src/Ports.integration.test.ts", + "packages/stack/src/port-reservation.e2e.test.ts", + "packages/stack/src/StackHost.ts", + "packages/stack/src/host/CommandRunner.ts", + "packages/stack/src/preparation/ArtifactStore.ts", + "packages/stack/src/preparation/artifacts.integration.test.ts", + "packages/stack/src/runtime/Container.ts", + "packages/stack/src/runtime/NativeProcess.ts", + "packages/stack/src/runtime/native-process.integration.test.ts", + "packages/stack/src/runtime/native-launcher.ts", + "packages/stack/src/runtime/postgres-user.ts", + "packages/stack/src/services/Catalog.ts", + "packages/stack/src/services/Database.ts", + "packages/stack/src/services/DatabaseSnapshot.ts", + "packages/stack/src/services/ProcessRecipe.ts", + "packages/stack/src/services/Recipe.ts", + "packages/stack/src/storage/DockerDatabaseStorage.ts", + "packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts", + "packages/stack/tests/claims.ts" + ], + "rules": { + "no-restricted-imports": [ + "error", + { + "patterns": [ + { + "group": ["**/namespace/**"], + "message": "Namespace internals are private; consume them through StackNamespace.ts." + } + ], + "paths": [ + { + "name": "node:fs", + "message": "Raw node:fs is confined to namespace drivers and a short enumerated list of existing files; use Effect's FileSystem or a namespace driver." + }, + { + "name": "node:fs/promises", + "message": "Raw node:fs/promises is confined to namespace drivers and a short enumerated list of existing files; use Effect's FileSystem or a namespace driver." + }, + { + "name": "node:sqlite", + "allowTypeImports": true, + "message": "Raw node:sqlite stays in namespace/drivers (plus runtime/native-launcher.ts's standalone boundary); use a namespace driver." + }, + { + "name": "bun:sqlite", + "allowTypeImports": true, + "message": "Raw bun:sqlite stays in namespace/drivers (plus runtime/native-launcher.ts's standalone boundary); use a namespace driver." + } + ] + } + ] + } + }, + { + // `StackNamespace.ts` is the facade itself: it needs namespace imports, but is still an + // ordinary consumer of raw primitives, so those restrictions stay on. + "files": ["packages/stack/src/StackNamespace.ts"], + "rules": { + "no-restricted-imports": [ + "error", + { + "paths": [ + { + "name": "node:fs", + "message": "Raw node:fs is confined to namespace drivers and a short enumerated list of existing files; use Effect's FileSystem or a namespace driver." + }, + { + "name": "node:fs/promises", + "message": "Raw node:fs/promises is confined to namespace drivers and a short enumerated list of existing files; use Effect's FileSystem or a namespace driver." + }, + { + "name": "node:sqlite", + "allowTypeImports": true, + "message": "Raw node:sqlite stays in namespace/drivers (plus runtime/native-launcher.ts's standalone boundary); use a namespace driver." + }, + { + "name": "bun:sqlite", + "allowTypeImports": true, + "message": "Raw bun:sqlite stays in namespace/drivers (plus runtime/native-launcher.ts's standalone boundary); use a namespace driver." + } + ] + } + ] + } + }, + { + // Inside the namespace, drivers are the only place for raw `node:fs`/`node:sqlite`; sibling + // namespace modules import each other freely, so only the raw-primitive restriction repeats + // here (the facade exception above excludes this whole directory from that one). + "files": ["packages/stack/src/namespace/**"], + "excludeFiles": [ + "packages/stack/src/namespace/drivers/**", + "packages/stack/src/namespace/Namespace.integration.test.ts" + ], + "rules": { + "no-restricted-imports": [ + "error", + { + "paths": [ + { "name": "node:fs", "message": "Raw node:fs stays in namespace/drivers." }, + { + "name": "node:fs/promises", + "message": "Raw node:fs/promises stays in namespace/drivers." + }, + { + "name": "node:sqlite", + "allowTypeImports": true, + "message": "Raw node:sqlite stays in namespace/drivers." + }, + { + "name": "bun:sqlite", + "allowTypeImports": true, + "message": "Raw bun:sqlite stays in namespace/drivers." + } + ] + } + ] + } + }, + { + // Sqlite is the driver directory's whole purpose, so it stays allowed everywhere in it; raw + // `node:fs` is narrower, limited to `FileSystem.ts` (the no-follow lstat/rmdir driver) by + // exact filename, excluded below. + "files": ["packages/stack/src/namespace/drivers/**"], + "excludeFiles": ["packages/stack/src/namespace/drivers/FileSystem.ts"], + "rules": { + "no-restricted-imports": [ + "error", + { + "paths": [ + { "name": "node:fs", "message": "Raw node:fs stays in drivers/FileSystem.ts." }, + { + "name": "node:fs/promises", + "message": "Raw node:fs/promises stays in drivers/FileSystem.ts." + } + ] + } + ] + } + }, + { + // Synchronous existence check between manual scheduler steps; keeps the disable comment's + // reason, only sqlite stays off-limits. + "files": ["packages/stack/src/namespace/Namespace.integration.test.ts"], + "rules": { + "no-restricted-imports": [ + "error", + { + "paths": [ + { + "name": "node:sqlite", + "allowTypeImports": true, + "message": "Raw node:sqlite stays in namespace/drivers." + }, + { + "name": "bun:sqlite", + "allowTypeImports": true, + "message": "Raw bun:sqlite stays in namespace/drivers." + } + ] + } + ] + } + }, + { + // Existing direct namespace consumers the facade doesn't cover yet (tracked for step 3); + // none of them touch raw node:fs/node:sqlite themselves, so that restriction stays on. + // - Network.ts, Ports.ts (+its integration test), port-reservation.e2e.test.ts: read/reserve + // ports through PortReservations, not yet on the facade. + // - Owner.ts: needs Claims, namespaceError, no-follow lstatPath, and owned/borrowed Paths. + // - StackHost.ts: reads Claims. + // - ArtifactStore.ts, artifacts.integration.test.ts: use the sqlite-pin driver and Pin; + // artifact pins are not on the facade yet. + // - Container.ts, Database.ts, ProcessRecipe.ts, Catalog.ts: need owned/borrowed Paths + // and/or Environment. + // - NativeProcess.ts, native-process.integration.test.ts: need Environment. + // - postgres-user.ts: needs the no-follow lstatPath. + // - Recipe.ts: needs Claims types only. + // - DockerDatabaseStorage.ts (+its integration test): needs Claims types and Publication. + // - tests/claims.ts: a no-op `ContainerClaims`/`DirectoryClaims` fixture for tests that don't + // exercise the namespace's own reconcile loop. + "files": [ + "packages/stack/src/Network.ts", + "packages/stack/src/Owner.ts", + "packages/stack/src/Ports.ts", + "packages/stack/src/Ports.integration.test.ts", + "packages/stack/src/port-reservation.e2e.test.ts", + "packages/stack/src/StackHost.ts", + "packages/stack/src/preparation/ArtifactStore.ts", + "packages/stack/src/preparation/artifacts.integration.test.ts", + "packages/stack/src/runtime/Container.ts", + "packages/stack/src/runtime/NativeProcess.ts", + "packages/stack/src/runtime/native-process.integration.test.ts", + "packages/stack/src/runtime/postgres-user.ts", + "packages/stack/src/services/Catalog.ts", + "packages/stack/src/services/Database.ts", + "packages/stack/src/services/ProcessRecipe.ts", + "packages/stack/src/services/Recipe.ts", + "packages/stack/src/storage/DockerDatabaseStorage.ts", + "packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts", + "packages/stack/tests/claims.ts" + ], + "rules": { + "no-restricted-imports": [ + "error", + { + "paths": [ + { + "name": "node:fs", + "message": "Raw node:fs is confined to namespace drivers and a short enumerated list of existing files; use Effect's FileSystem or a namespace driver." + }, + { + "name": "node:fs/promises", + "message": "Raw node:fs/promises is confined to namespace drivers and a short enumerated list of existing files; use Effect's FileSystem or a namespace driver." + }, + { + "name": "node:sqlite", + "allowTypeImports": true, + "message": "Raw node:sqlite stays in namespace/drivers (plus runtime/native-launcher.ts's standalone boundary); use a namespace driver." + }, + { + "name": "bun:sqlite", + "allowTypeImports": true, + "message": "Raw bun:sqlite stays in namespace/drivers (plus runtime/native-launcher.ts's standalone boundary); use a namespace driver." + } + ] + } + ] + } + }, + { + // `host/CommandRunner.ts` needs Claims, Environment, and borrow from the namespace, plus the + // raw `rmdir` Effect's FileSystem has no non-recursive equivalent for (see its existing + // disable comment); sqlite stays off-limits. + "files": ["packages/stack/src/host/CommandRunner.ts"], + "rules": { + "no-restricted-imports": [ + "error", + { + "paths": [ + { + "name": "node:sqlite", + "allowTypeImports": true, + "message": "Raw node:sqlite stays in namespace/drivers." + }, + { + "name": "bun:sqlite", + "allowTypeImports": true, + "message": "Raw bun:sqlite stays in namespace/drivers." + } + ] + } + ] + } + }, + { + // `services/DatabaseSnapshot.ts` opens its own sqlite lock file for the snapshot store, a + // concern independent of the namespace's own locking; it also needs the no-follow + // lstatPath. Raw node:fs stays off-limits. + "files": ["packages/stack/src/services/DatabaseSnapshot.ts"], + "rules": { + "no-restricted-imports": [ + "error", + { + "paths": [ + { + "name": "node:fs", + "message": "Raw node:fs is confined to namespace drivers and a short enumerated list of existing files; use Effect's FileSystem or a namespace driver." + }, + { + "name": "node:fs/promises", + "message": "Raw node:fs/promises is confined to namespace drivers and a short enumerated list of existing files; use Effect's FileSystem or a namespace driver." + } + ] + } + ] + } + }, + { + // Standalone process outside the Effect runtime, launched before any service is wired up: it + // calls the sqlite-pin driver and raw node:fs/node:sqlite directly by design (see its + // existing disable comments), so none of the namespace restrictions apply here. + "files": ["packages/stack/src/runtime/native-launcher.ts"], + "rules": { + "no-restricted-imports": "off" + } + }, + { + // Raw node:fs with an existing disable comment explaining why Effect's FileSystem can't do + // the job; none of these import namespace internals or raw sqlite. + "files": [ + "packages/stack/src/internal/host-process.ts", + "packages/stack/src/Sweep.integration.test.ts", + "packages/stack/src/storage/DirectoryCopy.ts", + "packages/stack/src/storage/DirectoryCopy.integration.test.ts", + "packages/stack/src/effect-timeout.integration.test.ts", + "packages/stack/src/HostProcess.ts", + "packages/stack/src/functions/serve-main-resolver.integration.test.ts", + "packages/stack/tests/effect-timeout.fixture.ts", + "packages/stack/tests/host-process-fixture.ts", + "packages/stack/tests/failing-owner-fixture.ts" + ], + "rules": { + "no-restricted-imports": [ + "error", + { + "patterns": [ + { + "group": ["**/namespace/**"], + "message": "Namespace internals are private; consume them through StackNamespace.ts." + } + ], + "paths": [ + { + "name": "node:sqlite", + "allowTypeImports": true, + "message": "Raw node:sqlite stays in namespace/drivers." + }, + { + "name": "bun:sqlite", + "allowTypeImports": true, + "message": "Raw bun:sqlite stays in namespace/drivers." + } + ] + } + ] + } + }, + { + // Standalone fixture process the lease waits on, outside any Effect runtime: it takes the + // same raw sqlite lock as the lease it waits for and reports over a raw fd (see its existing + // disable comments). It doesn't import namespace internals, so that restriction stays on. + "files": ["packages/stack/tests/lease-wait-fixture.ts"], + "rules": { + "no-restricted-imports": [ + "error", + { + "patterns": [ + { + "group": ["**/namespace/**"], + "message": "Namespace internals are private; consume them through StackNamespace.ts." + } + ] + } + ] + } } ] } diff --git a/packages/stack/src/namespace/PortReservations.ts b/packages/stack/src/namespace/PortReservations.ts index f44fafd6bc..fe9cb3fbf9 100644 --- a/packages/stack/src/namespace/PortReservations.ts +++ b/packages/stack/src/namespace/PortReservations.ts @@ -1,10 +1,10 @@ import { userInfo } from "node:os"; -import { DatabaseSync } from "node:sqlite"; +import type { DatabaseSync } from "node:sqlite"; import { NodeServices } from "@effect/platform-node"; import { Context, Effect, FileSystem, Layer, Path, Predicate, Stream } from "effect"; import { ChildProcess } from "effect/unstable/process"; import { namespaceError, type NamespaceError } from "./Capabilities.ts"; -import { errcode } from "./drivers/Sqlite.ts"; +import { errcode, openSharedConnection } from "./drivers/Sqlite.ts"; import { lstatPath } from "./drivers/FileSystem.ts"; /** Identifies the stack that owns a port reservation. */ @@ -86,37 +86,14 @@ const isHolderRow = (value: unknown): value is HolderRow => Predicate.hasProperty(value, "stack_id") && typeof value.stack_id === "string"; -/** - * One connection per resolved database path for this process's lifetime. POSIX advisory locks (and - * the SQLite locks built on them) belong to a process, not a descriptor: a second connection this - * process opened on the same path would not see the first one's transaction as contention, so every - * caller in this process shares this one connection instead of risking that coalescing. - */ -const connections = new Map(); - const connectionFor = (file: string): Effect.Effect => - Effect.suspend(() => { - const existing = connections.get(file); - if (existing !== undefined) return Effect.succeed(existing); - return Effect.try({ - try: () => { - const connection = new DatabaseSync(file); - try { - // Short and bounded: real contention between two of this user's processes is brief, and - // a blocking wait here is simpler than an async retry loop for a database this small. Set - // before the schema statement, so even first-time table creation waits out a racing peer - // instead of surfacing a spurious busy error. - connection.exec("PRAGMA busy_timeout = 2000"); - connection.exec(schema); - } catch (error) { - connection.close(); - throw error; - } - connections.set(file, connection); - return connection; - }, - catch: (cause) => namespaceError("open", cause), - }); + openSharedConnection(file, (connection) => { + // Short and bounded: real contention between two of this user's processes is brief, and + // a blocking wait here is simpler than an async retry loop for a database this small. Set + // before the schema statement, so even first-time table creation waits out a racing peer + // instead of surfacing a spurious busy error. + connection.exec("PRAGMA busy_timeout = 2000"); + connection.exec(schema); }); /** diff --git a/packages/stack/src/namespace/drivers/Sqlite.ts b/packages/stack/src/namespace/drivers/Sqlite.ts index a0f8bd0a98..86544c7a1c 100644 --- a/packages/stack/src/namespace/drivers/Sqlite.ts +++ b/packages/stack/src/namespace/drivers/Sqlite.ts @@ -29,6 +29,39 @@ const closeConnection = (connection: DatabaseSync) => catch: (cause) => namespaceError("lock-close", cause), }); +/** + * One connection per resolved database path for a caller's process lifetime, running `init` only + * on first open. POSIX advisory locks (and the SQLite locks built on them) belong to a process, + * not a descriptor: a second connection this process opened on the same path would not see the + * first one's transaction as contention, so every caller sharing a path must share one connection + * instead of risking that coalescing. + */ +const sharedConnections = new Map(); + +/** Opens (or reuses) `path`'s process-lifetime shared connection; never closed until the process exits. */ +export const openSharedConnection = ( + path: string, + init: (connection: DatabaseSync) => void, +): Effect.Effect => + Effect.suspend(() => { + const existing = sharedConnections.get(path); + if (existing !== undefined) return Effect.succeed(existing); + return Effect.try({ + try: () => { + const connection = new DatabaseSync(path); + try { + init(connection); + } catch (error) { + connection.close(); + throw error; + } + sharedConnections.set(path, connection); + return connection; + }, + catch: (cause) => namespaceError("open", cause), + }); + }); + /** Matches a `NamespaceError`'s cause against one of `node:sqlite`'s numeric `errcode`s. */ export const errcode = (cause: unknown, code: number) => Predicate.hasProperty(cause, "errcode") && cause.errcode === code; From 0f9d06ef6afcc1b61082d74fe8859e0f77cc6340 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sun, 4 Oct 2026 09:03:07 +0200 Subject: [PATCH 020/142] test(stack): seed the database artifact so the native cleanup test never downloads or execs psql --- ...dRunner.native-cleanup.integration.test.ts | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/packages/stack/src/host/CommandRunner.native-cleanup.integration.test.ts b/packages/stack/src/host/CommandRunner.native-cleanup.integration.test.ts index 0a33d357c4..448abeea55 100644 --- a/packages/stack/src/host/CommandRunner.native-cleanup.integration.test.ts +++ b/packages/stack/src/host/CommandRunner.native-cleanup.integration.test.ts @@ -3,11 +3,70 @@ import { expect, it } from "@effect/vitest"; import { Context, Effect, FileSystem, Layer, Path, Stream } from "effect"; import { systemError } from "effect/PlatformError"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { resolveArtifact } from "../Artifacts.ts"; import { postgres } from "../Commands.ts"; +import { + makeArtifactStore, + type ArtifactRequest, + type ArtifactSource, +} from "../preparation/ArtifactStore.ts"; +import { PreparationError } from "../preparation/Errors.ts"; import * as CommandRunner from "./CommandRunner.ts"; import { noContainerClaims } from "../../tests/claims.ts"; -it.live.skipIf(process.platform === "win32")( +const target = + process.platform === "darwin" && process.arch === "arm64" + ? "darwin-arm64" + : process.platform === "linux" && process.arch === "x64" + ? "linux-amd64" + : process.platform === "linux" && process.arch === "arm64" + ? "linux-arm64" + : undefined; + +/** + * Publishes a trivial fixture in place of the real postgres artifact, at the exact cache key and + * digest `CommandRunner`'s own native launch resolves: its later `useNativeArtifact` call then + * hits this cached generation instead of downloading and running the real, much larger postgres + * distribution, whose first execution after a fresh extraction costs several additional seconds + * on top of the download (one-time code-signature validation of a freshly written binary). + */ +const prepareDatabaseArtifact = Effect.fn(function* (cacheRoot: string) { + if (target === undefined) return yield* Effect.fail("Unsupported test platform"); + const fs = yield* FileSystem.FileSystem; + const { releaseVersion, natives, requiredRuntimePaths, executablePath } = yield* resolveArtifact({ + service: "database", + }); + const request: ArtifactRequest = { + key: `slim-services/postgres/${releaseVersion}/${target}`, + requiredRuntimePaths, + executablePath, + }; + const digest = natives[target].archive; + const source: ArtifactSource = { + checksum: () => Effect.succeed(digest), + materialize: (_request, destination) => + Effect.gen(function* () { + for (const relative of requiredRuntimePaths) { + const file = `${destination}/${relative}`; + yield* fs.makeDirectory(file.slice(0, file.lastIndexOf("/")), { recursive: true }); + yield* fs.writeFileString(file, `#!${process.execPath}\nprocess.exit(0);\n`); + yield* fs.chmod(file, 0o755); + } + }).pipe( + Effect.mapError( + (cause) => + new PreparationError({ + message: `Unable to write database command fixture: ${cause.message}`, + cause, + }), + ), + ), + }; + const store = yield* makeArtifactStore({ cacheRoot, source }); + yield* store.prepare(request); +}); + +it.live.skipIf(target === undefined || process.platform === "win32")( "retries failed native workload cleanup when the stack runner is cleaned up", () => Effect.scoped( @@ -16,6 +75,7 @@ it.live.skipIf(process.platform === "win32")( const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "native-runner-cleanup-" }); const cacheRoot = path.join(root, "cache"); + yield* prepareDatabaseArtifact(cacheRoot); let isRunningCalls = 0; const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const spawner = ChildProcessSpawner.make((command) => From 871140cd5369a5516cf9f98ca547b7ecd57adb4f Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sun, 4 Oct 2026 09:52:25 +0200 Subject: [PATCH 021/142] fix(stack): reserve native backend ports below the ephemeral range --- .../experimental/stack/start/SIDE_EFFECTS.md | 4 + packages/stack/README.md | 6 +- packages/stack/src/Owner.ts | 16 ++ packages/stack/src/Ports.integration.test.ts | 140 +++++++++++++++++- packages/stack/src/Ports.ts | 117 ++++++++++++++- .../stack/src/namespace/PortReservations.ts | 13 +- ...nalyticsVectorImgproxy.integration.test.ts | 7 +- .../services/AuthStorage.integration.test.ts | 7 +- .../src/services/Catalog.integration.test.ts | 7 +- packages/stack/src/services/Catalog.ts | 1 + .../services/Functions.integration.test.ts | 14 +- .../src/services/Mail.integration.test.ts | 7 +- .../src/services/Pooler.integration.test.ts | 7 +- .../ProcessRecipe.integration.test.ts | 28 +++- packages/stack/src/services/ProcessRecipe.ts | 75 ++++------ .../src/services/Realtime.integration.test.ts | 7 +- packages/stack/src/services/Recipe.ts | 3 + .../src/services/Rest.integration.test.ts | 7 +- .../src/services/Studio.integration.test.ts | 7 +- .../src/services/Vector.integration.test.ts | 7 +- packages/stack/tests/claims.ts | 6 + 21 files changed, 422 insertions(+), 64 deletions(-) diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index a518715048..1612393424 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -45,6 +45,10 @@ again reuses the same ports. A saved or explicitly requested port already held b occupied by a process outside the registry, fails start naming the conflict instead of picking a different port; only automatic allocation tries another candidate. On macOS a process can still win a narrow race against the pre-bind probe before a brand-new listener exists. +Native backend ports (not publicly exposed) are reserved from 10000–19999, disjoint from the public +auto range and below the OS ephemeral range on every supported platform; this range is not +configurable, and a host whose ephemeral range has been widened to overlap it reintroduces the +ephemeral-port race this reservation exists to avoid. For a new stack, `--runtime auto` selects Docker when `docker version` reaches its daemon, then native on Linux x64/arm64 and macOS arm64. The probe is bounded by 10 seconds. Without a reachable diff --git a/packages/stack/README.md b/packages/stack/README.md index 4550fc8077..5bf92c57b4 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -113,7 +113,11 @@ probe before the real bind narrows but cannot close the race with a foreign proc same window; two of this user's stacks can never collide, because the registry excludes them. An unmounted or otherwise temporarily unreachable state root looks identical to a deleted one and can be reclaimed the same way. The registry does not reserve across OS users; isolation between users' -stacks still relies only on the kernel refusing a second bind. +stacks still relies only on the kernel refusing a second bind. A native backend's own (never public) +ports are reserved directly from 10000–19999, disjoint from the public auto range and below the OS +ephemeral range on every supported platform; this is not configurable, and a host whose ephemeral +range has been widened to overlap 10000–19999 reintroduces the ephemeral-port race this reservation +exists to avoid. ## Composition and operation scope diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index 7444c3f70c..ee67ca736e 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -20,6 +20,8 @@ import type { Rpc, RpcGroup } from "effect/unstable/rpc"; import { failureMessage } from "./internal/failure-message.ts"; import * as Network from "./Network.ts"; import type { NetworkEndpoint, NetworkNamespace } from "./Network.ts"; +import * as PortReservations from "./namespace/PortReservations.ts"; +import { PortError } from "./Ports.ts"; import * as Orchestrator from "./Orchestrator.ts"; import type { CompositionConfig } from "./Orchestrator.ts"; import { @@ -247,6 +249,18 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { const crypto = Context.get(services, Crypto.Crypto); const path = Context.get(services, Path.Path); const network = yield* Network.Service; + const portReservations = yield* PortReservations.Service; + const isPubliclyReserved = (port: number) => + portReservations.isReserved(port).pipe( + Effect.mapError( + (cause) => + new PortError({ + key: "native", + message: "Unable to check the public port reservation registry", + cause, + }), + ), + ); const orchestrator = yield* Orchestrator.make(); const helpers = yield* makeDockerHelperRegistry(yield* crypto.randomUUIDv4); const definitionGate = yield* Semaphore.make(1); @@ -365,6 +379,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { helpers, containerClaims: claims.containers, directoryClaims: claims.directories, + isPubliclyReserved, ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), ...(options.engineTarget === undefined ? {} : { engineTarget: options.engineTarget }), }).pipe(Effect.provideContext(services)); @@ -828,4 +843,5 @@ export const layer = (options: Omit) => runtime: options.saved.runtime, }), ), + Layer.provide(PortReservations.layer), ); diff --git a/packages/stack/src/Ports.integration.test.ts b/packages/stack/src/Ports.integration.test.ts index 4457311ea4..f18a19c8cd 100644 --- a/packages/stack/src/Ports.integration.test.ts +++ b/packages/stack/src/Ports.integration.test.ts @@ -1,9 +1,27 @@ import { NodeFileSystem, NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Context, Effect, Exit, FileSystem, Fiber, Layer, Path, Schedule, Scope } from "effect"; +import { + Context, + Effect, + Exit, + FileSystem, + Fiber, + Hash, + Layer, + Path, + Schedule, + Scope, +} from "effect"; import * as Net from "node:net"; import { randomUUID } from "node:crypto"; -import { makePorts, PortError, probeVacant } from "./Ports.ts"; +import { + makePorts, + nativePortBase, + nativePortSpan, + PortError, + probeVacant, + reserveNativePort, +} from "./Ports.ts"; import { bindTcp } from "./Proxy.ts"; import * as PortReservations from "./namespace/PortReservations.ts"; import * as StackNamespace from "./StackNamespace.ts"; @@ -42,6 +60,47 @@ const saveStack = (state: StackNamespace.Interface, root: string, id: string) => const probedBind = (key: string) => (host: string, port: number) => probeVacant(process.platform)(key, host, port).pipe(Effect.andThen(bindTcp(host, port))); +/** Stands in for the real per-user registry check in tests that are not exercising it. */ +const neverPubliclyReserved = () => Effect.succeed(false); + +/** + * Binds a real wildcard listener directly in the native span, skipping any already-occupied + * candidate, instead of reserving a port through the allocator and racing to rebind it as a + * blocker afterward. + */ +const bindNativeWildcard = (): Effect.Effect<{ + readonly port: number; + readonly server: Net.Server; +}> => + Effect.gen(function* () { + for (let offset = 0; offset < nativePortSpan; offset++) { + const port = nativePortBase + offset; + const attempt = yield* Effect.exit( + Effect.callback((resume) => { + const server = Net.createServer(); + server.once("error", (cause) => resume(Effect.fail(cause))); + server.listen(port, "0.0.0.0", () => resume(Effect.succeed(server))); + }), + ); + if (Exit.isSuccess(attempt)) return { port, server: attempt.value }; + } + return yield* Effect.die("No port in the native span was free for the fixture"); + }); + +/** + * Brute-forces a key whose hash-seeded first scan candidate is exactly `port`, so a test can plant + * a blocker there ahead of time and assert the scan skips it, rather than merely missing it by + * chance. + */ +const keyWithFirstCandidate = (port: number): string => { + const offset = port - nativePortBase; + for (let attempt = 0; attempt < nativePortSpan * 20; attempt++) { + const key = `native-fixture-${attempt}`; + if (Math.abs(Hash.string(key)) % nativePortSpan === offset) return key; + } + throw new Error(`No key found whose first candidate is port ${port}`); +}; + it.live( "on this platform, auto allocation skips a port a foreign listener occupies, even where overlapping binds would otherwise succeed", () => @@ -270,3 +329,80 @@ it.live( ), ), ); + +it.live("reserveNativePort skips a port a wildcard listener already holds", () => + Effect.scoped( + Effect.gen(function* () { + const blocked = yield* bindNativeWildcard(); + yield* Effect.addFinalizer(() => + Effect.callback((resume) => { + blocked.server.close(() => resume(Effect.void)); + }), + ); + // A key whose hash-seeded first candidate is exactly the blocked port, so the scan must skip + // past the wildcard listener instead of merely missing it by chance. + const key = keyWithFirstCandidate(blocked.port); + + const reserved = yield* reserveNativePort(key, new Set(), neverPubliclyReserved); + expect(reserved.port).not.toBe(blocked.port); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("reserveNativePort excludes a port a previous attempt lost from the next reservation", () => + Effect.scoped( + Effect.gen(function* () { + const key = `native-exclude-${randomUUID()}`; + const firstScope = yield* Scope.make(); + const first = yield* reserveNativePort(key, new Set(), neverPubliclyReserved).pipe( + Effect.provideService(Scope.Scope, firstScope), + ); + yield* Scope.close(firstScope, Exit.void); + + const second = yield* reserveNativePort(key, new Set([first.port]), neverPubliclyReserved); + expect(second.port).not.toBe(first.port); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live( + "reserveNativePort skips a stopped stack's fixed public reservation inside the native range", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const id = `native-public-${randomUUID()}`; + const state = yield* makeTestState(root); + yield* saveStack(state, root, id); + const portReservations = yield* PortReservations.Service; + const stateRoot = yield* fs.realPath(root); + + // A free candidate in the native span, claimed as stack `id`'s durable fixed public + // reservation with no live listener: the stopped-but-not-destroyed state the registry + // keeps. The row, not a socket, is what the native allocator must respect here. + const target = yield* bindNativeWildcard(); + yield* Effect.callback((resume) => { + target.server.close(() => resume(Effect.void)); + }); + const conflict = yield* portReservations.reserve(stateRoot, id, "fixed", target.port); + yield* Effect.addFinalizer(() => + portReservations.release(stateRoot, id, "fixed").pipe(Effect.orDie), + ); + expect(conflict).toBeUndefined(); + + const key = keyWithFirstCandidate(target.port); + const isPubliclyReserved = (port: number) => + portReservations.isReserved(port).pipe(Effect.orDie); + const reserved = yield* reserveNativePort(key, new Set(), isPubliclyReserved); + expect(reserved.port).not.toBe(target.port); + }), + ).pipe( + Effect.provide( + Layer.merge( + NodeServices.layer, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), +); diff --git a/packages/stack/src/Ports.ts b/packages/stack/src/Ports.ts index d38f54dcfb..0e5fa10298 100644 --- a/packages/stack/src/Ports.ts +++ b/packages/stack/src/Ports.ts @@ -125,11 +125,126 @@ export const probeVacant = if (yield* loopbackVacant(platform)(port)) return; return yield* new PortError({ key, - message: `Public port ${port} for ${key} at ${host}:${port} is already in use`, + message: `Port ${port} for ${key} at ${host}:${port} is already in use`, conflict: { port, endpoint: key, holder: "foreign" }, }); }); +/** + * Disjoint from the public auto range (`portBase`..`portBase + portSpan`) and contiguous below it, + * so a native backend's direct bind can never land on a port the per-user registry is reserving for + * a public listener. Both ranges stay below the OS ephemeral range (ADR 0017). + */ +export const nativePortBase = 10000; +export const nativePortSpan = portBase - nativePortBase; +/** Co-prime with the span, matching the public scan's stride. */ +const nativePortStride = 257; + +export interface NativePortReservation { + readonly port: number; + readonly server: Net.Server; +} + +const closeNativePort = (server: Net.Server): Effect.Effect => + Effect.callback((resume) => { + if (!server.listening) { + resume(Effect.void); + return Effect.void; + } + server.close(() => resume(Effect.void)); + return Effect.void; + }); + +const bindNativePort = ( + key: string, + port: number, +): Effect.Effect => + Effect.callback((resume) => { + const server = Net.createServer((socket) => socket.destroy()); + const onError = (cause: Error) => + resume( + Effect.fail( + new PortError({ key, message: "Unable to reserve native service port", cause }), + ), + ); + server.once("error", onError); + server.listen({ host: "127.0.0.1", port }, () => { + const address = server.address(); + if (address === null || typeof address === "string") { + onError(new Error("Native service port reservation returned no address")); + } else { + resume(Effect.succeed({ port: address.port, server })); + } + }); + return Effect.sync(() => { + server.off("error", onError); + if (server.listening) server.close(); + }); + }); + +/** + * Reserves a native backend's private port, hash-seeding the scan from `key` so a reopened + * instance starts from the same candidate while separate keys spread out (mirrors the public auto + * scan's hash seed). Skips a candidate `isPubliclyReserved` reports, so a native bind can never + * take a port the per-user registry holds for a stopped stack's public listener (that stack owns + * no live socket to probe or bind against). Otherwise probes with {@link probeVacant} before the + * real bind, since a loopback-only bind can silently coexist with a wildcard listener on macOS, + * BSD, and Windows. A port a prior attempt in this batch lost is passed in `excluded` so a retry + * advances instead of repeating it. Never saved: backend ports are private and are not stable + * across restarts. + */ +export const reserveNativePort = Effect.fn("Ports.reserveNativePort")( + ( + key: string, + excluded: ReadonlySet, + isPubliclyReserved: (port: number) => Effect.Effect, + platform: NodeJS.Platform = process.platform, + ): Effect.Effect => + Effect.acquireRelease( + Effect.gen(function* () { + const probe = probeVacant(platform); + const start = Math.abs(Hash.string(key)) % nativePortSpan; + let failures = 0; + let lastFailure: PortError | undefined; + for (let attempt = 0; attempt < nativePortSpan && failures < 64; attempt++) { + const port = nativePortBase + ((start + attempt * nativePortStride) % nativePortSpan); + if (excluded.has(port)) continue; + if (yield* isPubliclyReserved(port)) { + failures++; + lastFailure = new PortError({ + key, + message: `Port ${port} is reserved by a public listener`, + }); + continue; + } + const probed = yield* Effect.exit(probe(key, "127.0.0.1", port)); + if (Exit.isFailure(probed)) { + const probeError = Cause.findErrorOption(probed.cause); + if (Option.isNone(probeError)) return yield* Effect.failCause(probed.cause); + failures++; + lastFailure = probeError.value; + continue; + } + const result = yield* Effect.exit(bindNativePort(key, port)); + if (Exit.isSuccess(result)) return result.value; + const error = Cause.findErrorOption(result.cause); + if (Option.isNone(error)) return yield* Effect.failCause(result.cause); + failures++; + lastFailure = error.value; + } + return yield* new PortError({ + key, + message: + lastFailure === undefined + ? "No native service port is available" + : `No native service port is available: ${lastFailure.message}`, + cause: lastFailure, + }); + }), + ({ server }) => closeNativePort(server), + ), +); + const resolveRequest = ( stack: StackNamespace.SavedStack, request: PortRequest, diff --git a/packages/stack/src/namespace/PortReservations.ts b/packages/stack/src/namespace/PortReservations.ts index fe9cb3fbf9..6eb2f121ae 100644 --- a/packages/stack/src/namespace/PortReservations.ts +++ b/packages/stack/src/namespace/PortReservations.ts @@ -22,6 +22,8 @@ export interface Interface { stackId: string, endpoint: string, ) => Effect.Effect; + /** True when any stack, at any state root, holds a public reservation for `port`. */ + readonly isReserved: (port: number) => Effect.Effect; /** * Reserves `port` for `(stateRoot, stackId, endpoint)`. Resolves to `undefined` once the row is * committed, or the live {@link Holder} already occupying `port` when it is taken. @@ -240,6 +242,15 @@ const make = Effect.fn("PortReservations.make")(function* (): Effect.fn.Return< return isHolderRow(row) ? row : undefined; }; + // Untraced: the native backend scan in `Ports.ts` calls this once per candidate port, and its + // own span already records the attempt and failure counts for the whole scan. + const isReserved = Effect.fnUntraced(function* (port: number) { + return yield* Effect.try({ + try: () => holderOf(port) !== undefined, + catch: (cause) => namespaceError("find", cause), + }); + }); + // Untraced: `Ports.ts`'s auto scan calls `reserve` and `reclaim` once per candidate port, and its // own span already records the attempt and failure counts for the whole scan. const reserve = Effect.fnUntraced(function* ( @@ -351,7 +362,7 @@ const make = Effect.fn("PortReservations.make")(function* (): Effect.fn.Return< }); }); - return { find, reserve, reclaim, release, releaseStack }; + return { find, isReserved, reserve, reclaim, release, releaseStack }; }); export const layer = Layer.effect(Service, make().pipe(Effect.map(Service.of))); diff --git a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts index c734cd2826..314151097c 100644 --- a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts +++ b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts @@ -6,7 +6,11 @@ import { makeService } from "../Service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { + noContainerClaims, + noDirectoryClaims, + noPublicPortReservations, +} from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ @@ -17,6 +21,7 @@ const options = (root: string) => ({ runtime: "native" as const, containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, }); const dockerOptions = (root: string) => ({ diff --git a/packages/stack/src/services/AuthStorage.integration.test.ts b/packages/stack/src/services/AuthStorage.integration.test.ts index 453562fb83..7becca6f13 100644 --- a/packages/stack/src/services/AuthStorage.integration.test.ts +++ b/packages/stack/src/services/AuthStorage.integration.test.ts @@ -7,7 +7,11 @@ import { makeService } from "../Service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { + noContainerClaims, + noDirectoryClaims, + noPublicPortReservations, +} from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ @@ -18,6 +22,7 @@ const options = (root: string) => ({ runtime: "native" as const, containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, }); const dockerOptions = (root: string) => ({ diff --git a/packages/stack/src/services/Catalog.integration.test.ts b/packages/stack/src/services/Catalog.integration.test.ts index 016b8e7909..8683d6eea2 100644 --- a/packages/stack/src/services/Catalog.integration.test.ts +++ b/packages/stack/src/services/Catalog.integration.test.ts @@ -2,7 +2,11 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Effect, Exit, FileSystem, Layer, Redacted } from "effect"; import { makeServiceRecipe } from "./Catalog.ts"; -import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { + noContainerClaims, + noDirectoryClaims, + noPublicPortReservations, +} from "../../tests/claims.ts"; const options = (root: string) => ({ stackId: "catalog-recipe", @@ -12,6 +16,7 @@ const options = (root: string) => ({ runtime: "native" as const, containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, }); describe("service catalog", () => { diff --git a/packages/stack/src/services/Catalog.ts b/packages/stack/src/services/Catalog.ts index 7957a293ef..d9a7eaa119 100644 --- a/packages/stack/src/services/Catalog.ts +++ b/packages/stack/src/services/Catalog.ts @@ -369,6 +369,7 @@ export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( spawner, container, borrowCallerPath, + isPubliclyReserved: options.isPubliclyReserved, }; switch (creation.service) { case "rest": diff --git a/packages/stack/src/services/Functions.integration.test.ts b/packages/stack/src/services/Functions.integration.test.ts index c1462c2321..a3a6be472e 100644 --- a/packages/stack/src/services/Functions.integration.test.ts +++ b/packages/stack/src/services/Functions.integration.test.ts @@ -22,7 +22,11 @@ import { makeService } from "../Service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import * as Functions from "./Functions.ts"; import { makeProcessRecipe } from "./ProcessRecipe.ts"; -import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { + noContainerClaims, + noDirectoryClaims, + noPublicPortReservations, +} from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ @@ -104,6 +108,7 @@ describe("service catalog", () => { { containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, ...dockerOptions(root), stackId, instanceId, @@ -168,6 +173,7 @@ describe("service catalog", () => { { containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, ...options(root), stackId, instanceId, @@ -263,6 +269,7 @@ describe("service catalog", () => { { containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, ...options(stackRoot), stackId: "d".repeat(64), instanceId: "ancestor", @@ -336,6 +343,7 @@ describe("service catalog", () => { { containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, ...options(stackRoot), stackId: "e".repeat(64), instanceId: "deno-config", @@ -412,6 +420,7 @@ describe("service catalog", () => { { containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, ...options(stackRoot), stackId: "f".repeat(64), instanceId: "plain-deno-config", @@ -527,6 +536,7 @@ for (const runtime of ["native", "docker"] as const) { { containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, ...options(stackRoot), stackId: "c".repeat(64), instanceId: "configured", @@ -637,6 +647,7 @@ it.effect("passes POSIX project paths to a docker Functions container from a Win }; const deps = { borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), + isPubliclyReserved: noPublicPortReservations, fs: yield* FileSystem.FileSystem, path: yield* Path.Path.pipe(Effect.provide(NodePath.layerWin32)), crypto: yield* Crypto.Crypto, @@ -649,6 +660,7 @@ it.effect("passes POSIX project paths to a docker Functions container from a Win { containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, stackId: "e".repeat(64), instanceId: "windows", root: "C:\\Users\\dev\\stack", diff --git a/packages/stack/src/services/Mail.integration.test.ts b/packages/stack/src/services/Mail.integration.test.ts index e445ef4633..c19305e8dc 100644 --- a/packages/stack/src/services/Mail.integration.test.ts +++ b/packages/stack/src/services/Mail.integration.test.ts @@ -4,7 +4,11 @@ import { Effect, Exit, FileSystem, Layer, Path } from "effect"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; import { makeService } from "../Service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; -import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { + noContainerClaims, + noDirectoryClaims, + noPublicPortReservations, +} from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ @@ -15,6 +19,7 @@ const options = (root: string) => ({ runtime: "native" as const, containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, }); const dockerOptions = (root: string) => ({ diff --git a/packages/stack/src/services/Pooler.integration.test.ts b/packages/stack/src/services/Pooler.integration.test.ts index 8b8c740536..e3113642a0 100644 --- a/packages/stack/src/services/Pooler.integration.test.ts +++ b/packages/stack/src/services/Pooler.integration.test.ts @@ -7,7 +7,11 @@ import { ProxyError } from "../Proxy.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { + noContainerClaims, + noDirectoryClaims, + noPublicPortReservations, +} from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ @@ -18,6 +22,7 @@ const options = (root: string) => ({ runtime: "native" as const, containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, }); const dockerOptions = (root: string) => ({ diff --git a/packages/stack/src/services/ProcessRecipe.integration.test.ts b/packages/stack/src/services/ProcessRecipe.integration.test.ts index 77bc573803..29724fbead 100644 --- a/packages/stack/src/services/ProcessRecipe.integration.test.ts +++ b/packages/stack/src/services/ProcessRecipe.integration.test.ts @@ -25,6 +25,7 @@ import * as Net from "node:net"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- the collision fixture owns a local HTTP listener. import * as NodeHttp from "node:http"; import { catalogPins, resolveArtifact, type ServiceKind } from "../Artifacts.ts"; +import { nativePortBase, nativePortSpan } from "../Ports.ts"; import { makeArtifactStore, type ArtifactRequest, @@ -46,7 +47,11 @@ import { import type { CatalogOptions, RecipeCreation } from "./Recipe.ts"; import * as Realtime from "./Realtime.ts"; import * as Pooler from "./Pooler.ts"; -import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { + noContainerClaims, + noDirectoryClaims, + noPublicPortReservations, +} from "../../tests/claims.ts"; type TestCreation = RecipeCreation<"rest", Record> & { readonly service: "rest"; @@ -76,6 +81,7 @@ const options: CatalogOptions = { runtime: "docker", containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, }; const spec: ProcessRecipeSpec = { @@ -192,6 +198,7 @@ describe("ProcessRecipe launch cleanup", () => { spawner: yield* ChildProcessSpawner.ChildProcessSpawner, container, borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), + isPubliclyReserved: noPublicPortReservations, } satisfies ProcessDependencies; const recipe = yield* makeProcessRecipe(creation, options, dependencies, spec); const service = yield* makeService(recipe.definition, { @@ -280,6 +287,7 @@ describe("ProcessRecipe launch cleanup", () => { spawner, container: undefined, borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), + isPubliclyReserved: noPublicPortReservations, } satisfies ProcessDependencies; const recipe = yield* makeProcessRecipe(creation, nativeOptions, dependencies, nativeSpec); const service = yield* makeService(recipe.definition, { @@ -391,6 +399,7 @@ describe("ProcessRecipe launch cleanup", () => { spawner, container: undefined, borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), + isPubliclyReserved: noPublicPortReservations, } satisfies ProcessDependencies; const recipe = yield* makeProcessRecipe(creation, nativeOptions, dependencies, nativeSpec); const service = yield* makeService(recipe.definition, { @@ -406,6 +415,10 @@ describe("ProcessRecipe launch cleanup", () => { expect(blocker?.listening).toBe(true); expect(ports).toHaveLength(3); expect(ports.every((port) => port > 0)).toBe(true); + for (const port of ports) { + expect(port).toBeGreaterThanOrEqual(nativePortBase); + expect(port).toBeLessThan(nativePortBase + nativePortSpan); + } yield* service.stop; }).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ), @@ -453,6 +466,7 @@ const realtimeService = Effect.fn(function* (container: ContainerRuntime) { { containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-test", instanceId: "instance", root: "/unused", @@ -461,6 +475,7 @@ const realtimeService = Effect.fn(function* (container: ContainerRuntime) { }, { borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), + isPubliclyReserved: noPublicPortReservations, fs: yield* FileSystem.FileSystem, path: yield* Path.Path, crypto: yield* Crypto.Crypto, @@ -670,6 +685,7 @@ const nativeRestRecipe = Effect.fn(function* ( }, { borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), + isPubliclyReserved: noPublicPortReservations, fs, path, crypto, @@ -940,6 +956,7 @@ describe("process recipe startup", () => { { containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-port-race", instanceId: "instance", root, @@ -949,6 +966,7 @@ describe("process recipe startup", () => { }, { borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), + isPubliclyReserved: noPublicPortReservations, fs, path, crypto, @@ -1017,6 +1035,7 @@ describe("process recipe startup", () => { { containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-port-exhaustion", instanceId: "instance", root, @@ -1026,6 +1045,7 @@ describe("process recipe startup", () => { }, { borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), + isPubliclyReserved: noPublicPortReservations, fs, path, crypto, @@ -1154,6 +1174,7 @@ describe("process recipe startup", () => { { containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-deadline", instanceId: "instance", root, @@ -1163,6 +1184,7 @@ describe("process recipe startup", () => { }, { borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), + isPubliclyReserved: noPublicPortReservations, fs, path, crypto, @@ -1219,6 +1241,7 @@ describe("process recipe startup", () => { { containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-unrelated-failure", instanceId: "instance", root, @@ -1228,6 +1251,7 @@ describe("process recipe startup", () => { }, { borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), + isPubliclyReserved: noPublicPortReservations, fs, path, crypto, @@ -1326,6 +1350,7 @@ describe("process recipe startup", () => { { containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-exit-failure", instanceId: "instance", root, @@ -1335,6 +1360,7 @@ describe("process recipe startup", () => { }, { borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), + isPubliclyReserved: noPublicPortReservations, fs, path, crypto, diff --git a/packages/stack/src/services/ProcessRecipe.ts b/packages/stack/src/services/ProcessRecipe.ts index 8d567327c0..9140d5f3e4 100644 --- a/packages/stack/src/services/ProcessRecipe.ts +++ b/packages/stack/src/services/ProcessRecipe.ts @@ -20,7 +20,6 @@ import { import { ChildProcessSpawner } from "effect/unstable/process"; import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; -import * as Net from "node:net"; import { prepareNativeArtifact, resolveArtifact, @@ -29,7 +28,7 @@ import { } from "../Artifacts.ts"; import * as Environment from "../namespace/Environment.ts"; import { containerInstancePath, destroyOwnedRoot, type BorrowedPath } from "../namespace/Paths.ts"; -import { accepts } from "../Ports.ts"; +import { accepts, reserveNativePort, type PortError } from "../Ports.ts"; import { type ContainerError, type ContainerProcess, @@ -187,6 +186,8 @@ export interface ProcessDependencies { readonly container: ContainerRuntime | undefined; /** Validates a caller-supplied path against the stack's data root; see `namespace/Paths.borrow`. */ readonly borrowCallerPath: (candidate: string) => Effect.Effect; + /** Read-only: true when the per-user registry holds `port` for any stack's public listener. */ + readonly isPubliclyReserved: (port: number) => Effect.Effect; } const serviceError = mapToServiceError; @@ -215,49 +216,7 @@ const runtimeFromNative = (process: NativeProcess): RuntimeSession => ({ remove: Effect.void, }); -interface NativePortReservation { - readonly port: number; - readonly server: Net.Server; -} - -const closeNativePort = (server: Net.Server): Effect.Effect => - Effect.callback((resume) => { - if (!server.listening) { - resume(Effect.void); - return Effect.void; - } - server.close(() => resume(Effect.void)); - return Effect.void; - }); - -const reserveNativePort = Effect.fn("ProcessRecipe.reserveNativePort")( - (requested: number): Effect.Effect => - Effect.acquireRelease( - Effect.callback((resume) => { - const server = Net.createServer((socket) => socket.destroy()); - const onError = (cause: Error) => - resume( - Effect.fail( - catalogError("launch", "Unable to reserve native service port", undefined, cause), - ), - ); - server.once("error", onError); - server.listen({ host: "127.0.0.1", port: requested }, () => { - const address = server.address(); - if (address === null || typeof address === "string") { - onError(new Error("Native service port reservation returned no address")); - } else { - resume(Effect.succeed({ port: address.port, server })); - } - }); - return Effect.sync(() => { - server.off("error", onError); - if (server.listening) server.close(); - }); - }), - ({ server }) => closeNativePort(server), - ), -); +const emptyNativePortSet: ReadonlySet = new Set(); const startupTimeoutSeconds = 60; const nativeLaunchAttempts = 3; @@ -529,13 +488,24 @@ export const makeProcessRecipe = const environment = yield* Environment.confine(deps.fs, deps.path, environmentRoot).pipe( Effect.mapError((cause) => serviceError("launch", cause)), ); + const keyFor = (name: string) => `${options.instanceId}:${context.id}:${name}`; + // A port a prior attempt lost stays excluded so a retry advances instead of repeating it. + const excludedByKey = yield* Ref.make>>(new Map()); const reserveEndpoints = Effect.fn("ProcessRecipe.reserveEndpoints")(function* ( parent: Scope.Closeable, ) { const portScope = yield* Scope.fork(parent, "sequential"); - const reservations = yield* Effect.forEach(portNames, () => reserveNativePort(0), { - concurrency: 1, - }).pipe( + const excluded = yield* Ref.get(excludedByKey); + const reservations = yield* Effect.forEach( + portNames, + ([name]) => + reserveNativePort( + keyFor(name), + excluded.get(keyFor(name)) ?? emptyNativePortSet, + deps.isPubliclyReserved, + ), + { concurrency: 1 }, + ).pipe( Scope.provide(portScope), Effect.mapError((cause) => serviceError("launch", cause)), ); @@ -755,6 +725,15 @@ export const makeProcessRecipe = (!(yield* Deferred.isDone(attempt.output.bindReady)) && (yield* anotherListenerHolds(attempt.selected)))); if (!collided) return yield* settleFailure(failure); + yield* Ref.update(excludedByKey, (current) => { + const next = new Map(current); + for (const [name, endpoint] of attempt.selected) { + if (endpoint.kind !== "tcp") continue; + const key = keyFor(name); + next.set(key, new Set([...(next.get(key) ?? []), endpoint.port])); + } + return next; + }); return yield* new NativePortCollision({ failure: serviceError( "launch", diff --git a/packages/stack/src/services/Realtime.integration.test.ts b/packages/stack/src/services/Realtime.integration.test.ts index bc7468628c..b60f506e77 100644 --- a/packages/stack/src/services/Realtime.integration.test.ts +++ b/packages/stack/src/services/Realtime.integration.test.ts @@ -6,7 +6,11 @@ import { makeService } from "../Service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { + noContainerClaims, + noDirectoryClaims, + noPublicPortReservations, +} from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ @@ -17,6 +21,7 @@ const options = (root: string) => ({ runtime: "native" as const, containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, }); const dockerOptions = (root: string) => ({ diff --git a/packages/stack/src/services/Recipe.ts b/packages/stack/src/services/Recipe.ts index 7479bf2533..018cfc2bca 100644 --- a/packages/stack/src/services/Recipe.ts +++ b/packages/stack/src/services/Recipe.ts @@ -4,6 +4,7 @@ import type { Effect, Ref } from "effect"; import type { ServiceKind } from "../Artifacts.ts"; import type { ServiceDefinition } from "../Service.ts"; import type { ContainerClaims, DirectoryClaims } from "../namespace/Claims.ts"; +import type { PortError } from "../Ports.ts"; import type { EngineTarget, HostGateway } from "../runtime/Container.ts"; import type { DockerHelperRegistry } from "../storage/DockerHelperRegistry.ts"; @@ -63,6 +64,8 @@ export interface CatalogOptions { readonly containerClaims: ContainerClaims; /** Journals directories a service creates outside its data root (for example under `/tmp`). */ readonly directoryClaims: DirectoryClaims; + /** Read-only: true when the per-user registry holds `port` for any stack's public listener. */ + readonly isPubliclyReserved: (port: number) => Effect.Effect; } export interface CatalogLog { diff --git a/packages/stack/src/services/Rest.integration.test.ts b/packages/stack/src/services/Rest.integration.test.ts index fbb5e9f2de..a9d0e9d0ef 100644 --- a/packages/stack/src/services/Rest.integration.test.ts +++ b/packages/stack/src/services/Rest.integration.test.ts @@ -11,7 +11,11 @@ import { makeService } from "../Service.ts"; import { ProxyError } from "../Proxy.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { + noContainerClaims, + noDirectoryClaims, + noPublicPortReservations, +} from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const makeTestState = (root: string) => @@ -38,6 +42,7 @@ const options = (root: string) => ({ runtime: "native" as const, containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, }); const dockerOptions = (root: string) => ({ diff --git a/packages/stack/src/services/Studio.integration.test.ts b/packages/stack/src/services/Studio.integration.test.ts index 2bfb4e945d..17974e0c42 100644 --- a/packages/stack/src/services/Studio.integration.test.ts +++ b/packages/stack/src/services/Studio.integration.test.ts @@ -3,7 +3,11 @@ import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Layer } from "effect"; import { makeService } from "../Service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; -import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { + noContainerClaims, + noDirectoryClaims, + noPublicPortReservations, +} from "../../tests/claims.ts"; const options = (root: string) => ({ stackId: "catalog-studio", @@ -13,6 +17,7 @@ const options = (root: string) => ({ runtime: "native" as const, containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, }); describe("studio recipe", () => { diff --git a/packages/stack/src/services/Vector.integration.test.ts b/packages/stack/src/services/Vector.integration.test.ts index 32412cac2a..d38026d953 100644 --- a/packages/stack/src/services/Vector.integration.test.ts +++ b/packages/stack/src/services/Vector.integration.test.ts @@ -5,7 +5,11 @@ import { Effect, FileSystem, Layer } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { makeService } from "../Service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; -import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { + noContainerClaims, + noDirectoryClaims, + noPublicPortReservations, +} from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string, runtime: "docker" | "native") => ({ @@ -17,6 +21,7 @@ const options = (root: string, runtime: "docker" | "native") => ({ ...(runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, + isPubliclyReserved: noPublicPortReservations, }); describe("vector recipe", () => { diff --git a/packages/stack/tests/claims.ts b/packages/stack/tests/claims.ts index 5b9bbb0a9a..a00913eb61 100644 --- a/packages/stack/tests/claims.ts +++ b/packages/stack/tests/claims.ts @@ -1,5 +1,6 @@ import { Effect } from "effect"; import type { ContainerClaims, DirectoryClaims } from "../src/namespace/Claims.ts"; +import type { PortError } from "../src/Ports.ts"; /** * No-op claims for tests that exercise container or directory creation without exercising the @@ -13,3 +14,8 @@ export const noDirectoryClaims: DirectoryClaims = { claim: () => Effect.void, unclaim: () => Effect.void, }; + +/** Stands in for the real per-user registry check in tests not exercising it (covered separately + * by `Ports.integration.test.ts`). */ +export const noPublicPortReservations = (_port: number): Effect.Effect => + Effect.succeed(false); From 76be6e684be26d5876316465bd8217f6c2d974e9 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sun, 4 Oct 2026 09:52:26 +0200 Subject: [PATCH 022/142] test(stack): identify native workload processes by marker, never by a reusable PID --- .../native-process.integration.test.ts | 64 ++++++++++++++----- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/packages/stack/src/runtime/native-process.integration.test.ts b/packages/stack/src/runtime/native-process.integration.test.ts index 4f86409acb..5d35bc46c1 100644 --- a/packages/stack/src/runtime/native-process.integration.test.ts +++ b/packages/stack/src/runtime/native-process.integration.test.ts @@ -277,7 +277,9 @@ describe("native process group cleanup", () => { try: async () => { const launcher = defaultNativeProcessLauncher(); const workloadMarker = `fd5-workload-${process.pid}`; - const child = spawnProcess(launcher.command, launcher.args, { + // Carried as a trailing argv entry so the launcher's own command line is checkable + // the same way as the workload's, without the launcher reading or acting on it. + const child = spawnProcess(launcher.command, [...launcher.args, workloadMarker], { detached: true, stdio: ["ignore", "pipe", "pipe", "pipe", "pipe", "pipe"], }); @@ -322,18 +324,40 @@ describe("native process group cleanup", () => { } return rows; }; - const killGroup = (pid: number | undefined) => { - if (pid === undefined) return; + // A recycled PID or PGID can belong to an unrelated process group (another user's, + // in the EPERM case); only a marker match proves it is still this test's own. + const groupCarriesMarker = (groupId: number): boolean => { try { - process.kill(-pid, "SIGKILL"); + return execFileSync("ps", ["-axo", "pgid=,command="], { encoding: "utf8" }) + .split(/\r?\n/) + .some((line) => { + const match = /^\s*(\d+)\s+(.*)$/.exec(line); + return ( + match !== null && + match[1] !== undefined && + match[2] !== undefined && + Number(match[1]) === groupId && + match[2].includes(workloadMarker) + ); + }); + } catch { + return false; + } + }; + const errorCode = (cause: unknown): string | undefined => + typeof cause === "object" && cause !== null && "code" in cause + ? String(cause.code) + : undefined; + const killGroup = (groupId: number | undefined) => { + if (groupId === undefined || !groupCarriesMarker(groupId)) return; + try { + process.kill(-groupId, "SIGKILL"); } catch (cause) { - if ( - typeof cause !== "object" || - cause === null || - !("code" in cause) || - cause.code !== "ESRCH" - ) - throw cause; + const code = errorCode(cause); + // A group that lost its marker between the check and the signal already exited; + // ESRCH or EPERM against a group that still carries it is a real failure. + if ((code === "ESRCH" || code === "EPERM") && !groupCarriesMarker(groupId)) return; + throw cause; } }; const killOwnedGroups = () => { @@ -516,6 +540,9 @@ describe("native process group cleanup", () => { () => Effect.scoped( Effect.gen(function* () { + // Identifies the descendant's own argv so a PID the kernel recycled for an unrelated + // process right after the descendant exited is never mistaken for a still-live member. + const descendantMarker = `native-process-descendant-${process.pid}`; const native = yield* spawnNativeProcess({ executable: process.execPath, environment: testEnvironment, @@ -524,10 +551,11 @@ describe("native process group cleanup", () => { "-e", [ "import { spawn } from 'node:child_process';", - "const descendant = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: ['ignore', 'inherit', 'inherit'] });", + "const descendant = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)', process.argv[1]], { stdio: ['ignore', 'inherit', 'inherit'] });", "process.stdout.write(`READY ${process.pid} ${descendant.pid}\\n`);", "setInterval(() => {}, 1000);", ].join("\n"), + descendantMarker, ], }); const output = yield* Effect.acquireRelease( @@ -557,11 +585,11 @@ describe("native process group cleanup", () => { const match = /READY \d+ (\d+)/.exec(yield* Ref.get(output.text)); expect(match).not.toBeNull(); if (match === null) return; - const [state, status] = yield* Effect.scoped( + const [psOutput, status] = yield* Effect.scoped( Effect.gen(function* () { const descendant = yield* ChildProcess.make( "/bin/ps", - ["-p", match[1] ?? "", "-o", "stat="], + ["-p", match[1] ?? "", "-o", "stat=,command="], { stdin: "ignore", stdout: "pipe", stderr: "ignore" }, ); return yield* Effect.all( @@ -570,9 +598,13 @@ describe("native process group cleanup", () => { ); }), ); + const line = psOutput.trim(); + const state = line.split(/\s+/u)[0] ?? ""; + // A recycled PID belongs to an unrelated process, not the descendant; its argv won't + // carry the marker, and that counts the same as the descendant already being gone. + const stillTracksDescendant = line.includes(descendantMarker); expect( - (Number(status) === 1 && state.trim() === "") || - (Number(status) === 0 && state.trim().startsWith("Z")), + Number(status) === 1 || line === "" || !stillTracksDescendant || state.startsWith("Z"), ).toBe(true); }), ).pipe(Effect.provide(NodeServices.layer)), From 9eb3cc3d423cf088983d3d6018b83e2368800d0a Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sun, 4 Oct 2026 11:36:14 +0200 Subject: [PATCH 023/142] fix(cli): name the changed setting and the fix when a saved stack rejects a config change --- apps/cli/src/command-internal/stack-config.ts | 197 +++- .../experimental/stack/start/SIDE_EFFECTS.md | 17 +- .../experimental/stack/start/start.command.ts | 6 +- .../experimental/stack/start/start.handler.ts | 504 +++++++++- .../stack/start/start.integration.test.ts | 913 +++++++++++++++++- packages/stack/package.json | 1 + packages/stack/src/internal/composition.ts | 2 + 7 files changed, 1530 insertions(+), 110 deletions(-) create mode 100644 packages/stack/src/internal/composition.ts diff --git a/apps/cli/src/command-internal/stack-config.ts b/apps/cli/src/command-internal/stack-config.ts index 266216daaa..bf8ddcd138 100644 --- a/apps/cli/src/command-internal/stack-config.ts +++ b/apps/cli/src/command-internal/stack-config.ts @@ -162,6 +162,142 @@ const envNestedPortOrConfigured = ( return typeof nested?.[key] === "number" ? configured : undefined; }; +/** A setting's `config.toml` key and the `SUPABASE_*` env var that overrides it. */ +export interface StackEndpointSetting { + readonly configPath: string; + readonly envVar: string; +} + +/** + * A port setting's `config.toml` key and env var, shared by `createCreations` and + * `stackEndpointSetting` so both report the same names; a new endpoint still needs an entry in + * `endpointSettingsByServiceEndpoint`. + */ +interface PortSetting extends StackEndpointSetting { + readonly section: string; + readonly nestedSection?: string; + readonly key: string; +} + +const DB_PORT: PortSetting = { + envVar: "SUPABASE_DB_PORT", + section: "db", + key: "port", + configPath: "db.port", +}; +const API_PORT: PortSetting = { + envVar: "SUPABASE_API_PORT", + section: "api", + key: "port", + configPath: "api.port", +}; +const STUDIO_PORT: PortSetting = { + envVar: "SUPABASE_STUDIO_PORT", + section: "studio", + key: "port", + configPath: "studio.port", +}; +const DB_POOLER_PORT: PortSetting = { + envVar: "SUPABASE_DB_POOLER_PORT", + section: "db", + nestedSection: "pooler", + key: "port", + configPath: "db.pooler.port", +}; +const LOCAL_SMTP_PORT: PortSetting = { + envVar: "SUPABASE_LOCAL_SMTP_PORT", + section: "local_smtp", + key: "port", + configPath: "local_smtp.port", +}; +const LOCAL_SMTP_SMTP_PORT: PortSetting = { + envVar: "SUPABASE_LOCAL_SMTP_SMTP_PORT", + section: "local_smtp", + key: "smtp_port", + configPath: "local_smtp.smtp_port", +}; +const LOCAL_SMTP_POP3_PORT: PortSetting = { + envVar: "SUPABASE_LOCAL_SMTP_POP3_PORT", + section: "local_smtp", + key: "pop3_port", + configPath: "local_smtp.pop3_port", +}; +const ANALYTICS_PORT: PortSetting = { + envVar: "SUPABASE_ANALYTICS_PORT", + section: "analytics", + key: "port", + configPath: "analytics.port", +}; +const ANALYTICS_VECTOR_PORT: PortSetting = { + envVar: "SUPABASE_ANALYTICS_VECTOR_PORT", + section: "analytics", + key: "vector_port", + configPath: "analytics.vector_port", +}; +const EDGE_RUNTIME_INSPECTOR_PORT: PortSetting = { + envVar: "SUPABASE_EDGE_RUNTIME_INSPECTOR_PORT", + section: "edge_runtime", + key: "inspector_port", + configPath: "edge_runtime.inspector_port", +}; + +/** Resolves one `PortSetting` against the loaded document and env, picking the nested variant when needed. */ +const resolvePort = ( + setting: PortSetting, + document: Readonly> | undefined, + configured: number, + env: Readonly>, +): number | undefined => + setting.nestedSection === undefined + ? envPortOrConfigured(setting.envVar, document, setting.section, setting.key, configured, env) + : envNestedPortOrConfigured( + setting.envVar, + document, + setting.section, + setting.nestedSection, + setting.key, + configured, + env, + ); + +/** + * Maps a saved stack endpoint (service + endpoint name) to the `PortSetting` that controls it. + * An endpoint missing here (e.g. `pooler.http`, `realtime.rpc`) is always automatic. + */ +const endpointSettingsByServiceEndpoint: Readonly> = { + "database.sql": DB_PORT, + "pooler.sql": DB_POOLER_PORT, + "analytics.http": ANALYTICS_PORT, + "vector.http": ANALYTICS_VECTOR_PORT, + "studio.http": STUDIO_PORT, + "mail.http": LOCAL_SMTP_PORT, + "mail.smtp": LOCAL_SMTP_SMTP_PORT, + "mail.pop3": LOCAL_SMTP_POP3_PORT, + "functions.inspector": EDGE_RUNTIME_INSPECTOR_PORT, + "rest.http": API_PORT, + "auth.http": API_PORT, + "realtime.http": API_PORT, + "storage.http": API_PORT, + "functions.http": API_PORT, +}; + +/** The config.toml key and env var override for a service endpoint, when the CLI exposes one. */ +export const stackEndpointSetting = ( + service: string, + endpoint: string, +): StackEndpointSetting | undefined => endpointSettingsByServiceEndpoint[`${service}.${endpoint}`]; + +/** + * `db.major_version`'s config key and `SUPABASE_DB_MAJOR_VERSION` override. Unlike the ports + * above, `envOverrideMajorVersion` (shared with `db-bootstrap` and the legacy local stack) hardcodes + * its own name/field, so there is no single call site to read this from without widening that + * shared helper's signature; the two literals here are kept in sync by hand. + */ +export const stackMajorVersionSetting: StackEndpointSetting = { + configPath: "db.major_version", + envVar: "SUPABASE_DB_MAJOR_VERSION", +}; + const authProviderNames = [ "apple", "azure", @@ -1070,68 +1206,51 @@ export const loadStackConfig = Effect.fn("StackConfig.load")( }, catch: (cause) => new StackConfigError({ message: String(cause) }), }); - const dbPort = envPortOrConfigured( - "SUPABASE_DB_PORT", + const dbPort = resolvePort( + DB_PORT, document, - "db", - "port", validatedConfig.db.port, context.projectEnvValues, ); - const apiPort = envPortOrConfigured( - "SUPABASE_API_PORT", + const apiPort = resolvePort( + API_PORT, document, - "api", - "port", validatedConfig.api.port, context.projectEnvValues, ); - const studioPort = envPortOrConfigured( - "SUPABASE_STUDIO_PORT", + const studioPort = resolvePort( + STUDIO_PORT, document, - "studio", - "port", validatedConfig.studio.port, context.projectEnvValues, ); - const poolerPort = envNestedPortOrConfigured( - "SUPABASE_DB_POOLER_PORT", + const poolerPort = resolvePort( + DB_POOLER_PORT, document, - "db", - "pooler", - "port", validatedConfig.db.pooler.port, context.projectEnvValues, ); - const mailPort = envPortOrConfigured( - "SUPABASE_LOCAL_SMTP_PORT", + const mailPort = resolvePort( + LOCAL_SMTP_PORT, document, - "local_smtp", - "port", validatedConfig.local_smtp.port, context.projectEnvValues, ); - const mailSmtpPort = envPortOrConfigured( - "SUPABASE_LOCAL_SMTP_SMTP_PORT", + const mailSmtpPort = resolvePort( + LOCAL_SMTP_SMTP_PORT, document, - "local_smtp", - "smtp_port", validatedConfig.local_smtp.smtp_port ?? 0, context.projectEnvValues, ); - const mailPop3Port = envPortOrConfigured( - "SUPABASE_LOCAL_SMTP_POP3_PORT", + const mailPop3Port = resolvePort( + LOCAL_SMTP_POP3_PORT, document, - "local_smtp", - "pop3_port", validatedConfig.local_smtp.pop3_port ?? 0, context.projectEnvValues, ); - const analyticsPort = envPortOrConfigured( - "SUPABASE_ANALYTICS_PORT", + const analyticsPort = resolvePort( + ANALYTICS_PORT, document, - "analytics", - "port", validatedConfig.analytics.port, context.projectEnvValues, ); @@ -1263,11 +1382,9 @@ export const loadStackConfig = Effect.fn("StackConfig.load")( config: { apiKey: "api-key" }, endpoints: { http: endpoint( - envPortOrConfigured( - "SUPABASE_ANALYTICS_VECTOR_PORT", + resolvePort( + ANALYTICS_VECTOR_PORT, document, - "analytics", - "vector_port", validatedConfig.analytics.vector_port ?? 0, context.projectEnvValues, ), @@ -1301,11 +1418,9 @@ export const loadStackConfig = Effect.fn("StackConfig.load")( endpoints: { http: endpoint(apiPort), inspector: endpoint( - envPortOrConfigured( - "SUPABASE_EDGE_RUNTIME_INSPECTOR_PORT", + resolvePort( + EDGE_RUNTIME_INSPECTOR_PORT, document, - "edge_runtime", - "inspector_port", validatedConfig.edge_runtime.inspector_port, context.projectEnvValues, ), diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index 1612393424..78113174b1 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -102,8 +102,9 @@ and ports are retained. Changed exclusions reuse existing service identities, da Removed services remain saved and stopped so including them again can reuse them; a saved stopped instance of a newly included service is reused when its endpoints and versions still match. The project configuration file is unchanged. A changed endpoint, artifact version, or PostgreSQL major -version fails before modifying the stopped composition, naming the changed setting and suggesting -`supabase stack destroy` to recreate the stack. +version fails before modifying the stopped composition, naming the `config.toml` key or +`SUPABASE_*` env var behind the change with its saved and requested values, and suggesting either +reverting it or running the stack's exact `supabase stack destroy` command to recreate it. ## First startup and retries @@ -151,3 +152,15 @@ name (protocol, address, port, and URL, matching `stack status`, with no synthet empty message. See [`docs/stack-commands.md`](../../../../../docs/stack-commands.md) for an example. Failures retain typed command errors and package diagnostics. Telemetry state is flushed after success or failure. + +A rejected configuration change additionally carries `stack_changes` on the JSON/stream-json error +envelope: one entry per affected service/path pair (a shared setting such as the API port appears +once per API-backed service even though the text message deduplicates it to one line; a service +with more than one incompatible path, such as a changed endpoint alongside an artifact version +change, gets one entry per path), each with `service`, `path` (the composition planner's dotted +path, e.g. `endpoints.http.port`, not a `config.toml` key — an added or dropped endpoint instead +reports `endpoints.`, since the planner only reports the bare `endpoints` path for that +case), `key`, `saved`, `requested`, and `editable`. `recreate_command` is the exact `supabase stack +destroy --stack-id ` invocation, without `--yes`, since destroy deletes local database data; +running it non-interactively or with `--output-format json`/`--output-format stream-json` requires +passing `--yes` explicitly. diff --git a/apps/cli/src/commands/experimental/stack/start/start.command.ts b/apps/cli/src/commands/experimental/stack/start/start.command.ts index e5a84466b8..8b569790ca 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.command.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.command.ts @@ -1,6 +1,8 @@ +import { Layer } from "effect"; import { Command, Flag } from "effect/unstable/cli"; import type * as CliCommand from "effect/unstable/cli/Command"; import { withJsonErrorHandling } from "../../../../shared/output/json-error-handling.ts"; +import { machineErrorContextLayer } from "../../../../shared/output/machine-error-context.layer.ts"; import { withCommandTelemetry } from "../../../../telemetry/command-telemetry.ts"; import { stringSliceFlag } from "../../../../command-internal/string-slice-flag.ts"; import { stdinLayer } from "../../../../shared/runtime/stdin.layer.ts"; @@ -66,5 +68,7 @@ export const stackStartCommand = Command.make("start", config).pipe( ), // `stackStart`'s bucket-seeding path satisfies `promptYesNo`'s `Stdin` requirement here even // though it always passes `yes: true`/`interactive: false` and never reaches the prompt. - Command.provide(stdinLayer), + // `machineErrorContextLayer` carries the structured incompatible-change payload onto the + // JSON/stream-json error envelope. + Command.provide(Layer.mergeAll(stdinLayer, machineErrorContextLayer)), ); diff --git a/apps/cli/src/commands/experimental/stack/start/start.handler.ts b/apps/cli/src/commands/experimental/stack/start/start.handler.ts index b8079078f8..f263ff634d 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.handler.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.handler.ts @@ -1,4 +1,4 @@ -import { defaultRuntime } from "@supabase/stack/internal/artifacts"; +import { catalogPins, defaultRuntime, postgresVersion } from "@supabase/stack/internal/artifacts"; import { connectionEnv, renderStackSummary, @@ -28,6 +28,7 @@ import { type StackError, } from "@supabase/stack/effect"; import { Output } from "../../../../shared/output/output.service.ts"; +import { MachineErrorContext } from "../../../../shared/output/machine-error-context.service.ts"; import { OutputFlag, resolveExperimentalWithProjectEnv, @@ -48,7 +49,13 @@ import { seedBucketsRun, } from "../../../../command-internal/seed-buckets.ts"; import { loadLocalProjectContext } from "../../../../command-internal/local-project-context.ts"; -import { loadStackConfig } from "../../../../command-internal/stack-config.ts"; +import { + loadStackConfig, + stackEndpointSetting, + stackMajorVersionSetting, + type StackEndpointSetting, +} from "../../../../command-internal/stack-config.ts"; +import { envOverride } from "../../../../command-internal/local-config-values.ts"; import { StackApi, stackCapabilityForService, @@ -165,23 +172,454 @@ const sameKinds = ( return leftKinds.size === rightKinds.size && [...leftKinds].every((kind) => rightKinds.has(kind)); }; -/** Rejects a saved instance whose endpoints or artifact versions the request would change. */ -const incompatibleChange = (planned: PlannedInstance) => - planned.change !== "incompatible" +/** + * Whether this set of requested services would run Studio without the REST API it depends on. + * The single source of truth for that dependency: both the real `--exclude` guard and the + * incompatible-change suggestion builder (which proposes a candidate exclusion set) validate + * against it, so a suggested `--exclude` can never itself fail this check. + */ +const studioNeedsRest = (requestedServices: ReadonlyArray<{ readonly service: string }>): boolean => + requestedServices.some(({ service }) => service === "studio") && + !requestedServices.some(({ service }) => service === "rest"); + +const isRecord = (value: unknown): value is Readonly> => + typeof value === "object" && value !== null; + +const endpointPortLabel = (endpoints: unknown, name: string): string => { + const intent = isRecord(endpoints) ? endpoints[name] : undefined; + const port = isRecord(intent) ? intent.port : undefined; + return port === "auto" ? "automatic" : typeof port === "number" ? String(port) : "unset"; +}; + +const endpointNamesOf = (endpoints: unknown): ReadonlyArray => + isRecord(endpoints) ? Object.keys(endpoints) : []; + +const databaseVersionOf = ( + creation: ServiceCreation | ServiceCreationInput | undefined, +): string | undefined => (creation?.service === "database" ? creation.config.version : undefined); + +const majorVersionOf = (version: string): string => version.split(".")[0] ?? version; + +/** + * The catalog's default pinned version for a service — what `resolveArtifact` selects when a + * creation's own top-level `version` is unset, which every CLI-built creation leaves unset. + */ +const defaultArtifactVersion = (service: PlannedInstance["service"]): string | undefined => + catalogPins().find((pin) => pin.service === service && pin.isDefault)?.pin.upstreamVersion; + +/** Renders a dotted config key as its `config.toml` section/key pair, e.g. `[db] major_version`. */ +const formatConfigPath = (path: string): string => { + const segments = path.split("."); + const key = segments.pop(); + return `[${segments.join(".")}] ${key}`; +}; + +/** The display key for a setting: its env var when that's what overrides it, else its config key. */ +const settingKeyLabel = ( + setting: StackEndpointSetting, + projectEnvValues: Readonly>, +): string => + envOverride(setting.envVar, undefined, projectEnvValues) !== undefined + ? setting.envVar + : formatConfigPath(setting.configPath); + +/** + * One incompatible path, reported as the JSON/stream-json error envelope's `stack_changes` + * entries (contract documented in `SIDE_EFFECTS.md`): one entry per affected service/path pair. + * `editable` marks whether `key` is a `config.toml` key or env var the user can revert, or plain + * wording for a catalog-pinned artifact, Postgres build, or endpoint presence change. + */ +interface StructuredSettingChange { + readonly service: string; + readonly path: string; + readonly key: string; + readonly saved: string; + readonly requested: string; + readonly editable: boolean; +} + +/** + * Why a change is non-editable, so the suggestion builder can pick accurate wording instead of + * re-deriving it from `StructuredSettingChange`'s display strings. Not part of the public + * `stack_changes` contract — only `change` below is. + */ +type ChangeReason = "setting" | "artifact" | "pinned-artifact" | "endpoint-presence"; + +interface DescribedChange { + readonly reason: ChangeReason; + readonly change: StructuredSettingChange; +} + +const settingChange = ( + reason: ChangeReason, + service: PlannedInstance["service"], + path: string, + key: string, + saved: string, + requested: string, + editable: boolean, +): DescribedChange => ({ reason, change: { service, path, key, saved, requested, editable } }); + +/** Whether a named endpoint is present at all, not what its port is. */ +const endpointPresenceChange = ( + service: PlannedInstance["service"], + name: string, + savedCreation: ServiceCreation | undefined, + requestedCreation: ServiceCreationInput | undefined, +): DescribedChange => { + const savedExposed = endpointNamesOf(savedCreation?.endpoints).includes(name); + const requestedExposed = endpointNamesOf(requestedCreation?.endpoints).includes(name); + return settingChange( + "endpoint-presence", + service, + `endpoints.${name}`, + `${service} ${name} endpoint`, + savedExposed ? "exposed" : "not exposed", + requestedExposed ? "exposed" : "not exposed", + false, + ); +}; + +/** + * Every endpoint added to, or dropped from, the request. The planner's bare `endpoints` path only + * says the two endpoint sets differ (e.g. a saved composition created with no public endpoint at + * all); this names exactly which endpoint(s) and in which direction, one entry per endpoint. + */ +const endpointPresenceChanges = ( + service: PlannedInstance["service"], + savedCreation: ServiceCreation | undefined, + requestedCreation: ServiceCreationInput | undefined, +): ReadonlyArray => { + const savedNames = endpointNamesOf(savedCreation?.endpoints); + const requestedNames = endpointNamesOf(requestedCreation?.endpoints); + const differing = [...new Set([...savedNames, ...requestedNames])].filter( + (name) => savedNames.includes(name) !== requestedNames.includes(name), + ); + return differing.map((name) => + endpointPresenceChange(service, name, savedCreation, requestedCreation), + ); +}; + +const describeSettingChange = ( + service: PlannedInstance["service"], + path: string, + savedCreation: ServiceCreation | undefined, + requestedCreation: ServiceCreationInput | undefined, + projectEnvValues: Readonly>, +): ReadonlyArray => { + if (service === "database" && path === "config.version") { + // `postgresVersion` resolves a bare major alias (e.g. "17") to the pinned build the + // composition plan actually compared, so the saved/requested pair reflects what changed. + const savedVersion = postgresVersion(databaseVersionOf(savedCreation) ?? "unknown"); + const requestedVersion = postgresVersion(databaseVersionOf(requestedCreation) ?? "unknown"); + const savedMajor = majorVersionOf(savedVersion); + const requestedMajor = majorVersionOf(requestedVersion); + // Same major but different pinned build: `major_version` doesn't control this, so reverting + // it wouldn't fix anything — name the actual (unpinnable) versions instead. + if (savedMajor === requestedMajor) + return [ + settingChange( + "artifact", + service, + path, + "Postgres build", + savedVersion, + requestedVersion, + false, + ), + ]; + return [ + settingChange( + "setting", + service, + path, + settingKeyLabel(stackMajorVersionSetting, projectEnvValues), + savedMajor, + requestedMajor, + true, + ), + ]; + } + // The planner reports the bare `endpoints` path when one side has no endpoints object at all + // (e.g. a saved composition created with no public endpoint): name which endpoint(s) the + // request adds or drops rather than misreporting this as an artifact/version change. + if (path === "endpoints") { + const presence = endpointPresenceChanges(service, savedCreation, requestedCreation); + if (presence.length > 0) return presence; + // The planner only reports this path when the endpoint sets genuinely differ; kept as a + // defensive fallback in case a future endpoint shape diffs without changing key sets. + return [ + settingChange("artifact", service, path, `${service} endpoints`, "changed", "changed", false), + ]; + } + if (path.startsWith("endpoints.")) { + const [endpointName, ...rest] = path.split(".").slice(1); + if (endpointName === undefined) + return [ + settingChange("artifact", service, path, `${service} ${path}`, "changed", "changed", false), + ]; + if (rest.length === 0) + // A two-segment `endpoints.` path (unlike the three-segment `endpoints..port` + // a port-value change produces) means the planner only found one side's endpoint object at + // all — this endpoint's presence differs, not its port. + return [endpointPresenceChange(service, endpointName, savedCreation, requestedCreation)]; + const setting = stackEndpointSetting(service, endpointName); + if (setting !== undefined) + return [ + settingChange( + "setting", + service, + path, + settingKeyLabel(setting, projectEnvValues), + endpointPortLabel(savedCreation?.endpoints, endpointName), + endpointPortLabel(requestedCreation?.endpoints, endpointName), + true, + ), + ]; + // A changed endpoint value with no config.toml key or env var override (e.g. `pooler.http`, + // `realtime.rpc`, always automatic): name it plainly instead of implying an editable setting. + return [ + settingChange("artifact", service, path, `${service} ${path}`, "changed", "changed", false), + ]; + } + if (path === "version") { + // CLI-built creations never set a top-level `version`, relying on the catalog's default: + // resolve that default for an unset side instead of reporting the raw (always-unset) field. + const defaultVersion = defaultArtifactVersion(service); + const savedVersion = savedCreation?.version ?? defaultVersion ?? "unknown"; + const requestedVersion = requestedCreation?.version ?? defaultVersion ?? "unknown"; + // The saved stack's explicit pin happens to equal what this release would select by default: + // there's no real artifact difference, so don't blame "this CLI release". + const reason: ChangeReason = savedVersion === requestedVersion ? "pinned-artifact" : "artifact"; + return [ + settingChange( + reason, + service, + path, + `${service} artifact version`, + savedVersion, + requestedVersion, + false, + ), + ]; + } + // Unreachable: the planner's own incompatible-path filter only yields "version", "endpoints", + // "endpoints.", or (database) "config.version". Kept for exhaustiveness. + return [ + settingChange("artifact", service, path, `${service} ${path}`, "changed", "changed", false), + ]; +}; + +/** Every incompatible path across every rejected saved member, as one structured list. */ +const incompatibleSettingChanges = ( + planned: ReadonlyArray, + savedConfigById: ReadonlyMap, + requested: ReadonlyArray, + projectEnvValues: Readonly>, +): ReadonlyArray => + planned + .filter((entry) => entry.member && entry.change === "incompatible") + .flatMap((entry) => + // Narrowed by the filter above; `Extract` isn't inferred through `.filter`. + entry.change === "incompatible" + ? entry.paths.flatMap((path) => + describeSettingChange( + entry.service, + path, + savedConfigById.get(entry.id), + requested.find((creation) => creation.service === entry.service), + projectEnvValues, + ), + ) + : [], + ); + +/** One deduplicated text line per distinct setting change (shared API port lines collapse to one). */ +const settingChangeLines = ( + changes: ReadonlyArray, +): ReadonlyArray => { + const seen = new Set(); + const lines: Array = []; + for (const change of changes) { + const line = `${change.key}: saved ${change.saved}, requested ${change.requested}`; + if (seen.has(line)) continue; + seen.add(line); + lines.push(line); + } + return lines; +}; + +const dedupe = (values: ReadonlyArray): ReadonlyArray => [...new Set(values)]; + +/** + * The exact `supabase stack destroy` invocation that recreates this stack. Always targets + * `--stack-id`: a `--stack ` destroy re-resolves the name against the caller's current + * `--workdir`, which can point at a different project's stack of the same name. Omits `--yes` on + * purpose, since destroying deletes local database data (details in `SIDE_EFFECTS.md`). + */ +const destroyCommandFor = (id: string): string => `supabase stack destroy --stack-id ${id}`; + +/** The revert clause for the editable keys among a rejection's changes, or `undefined` for none. */ +const revertAdvice = (editableKeys: ReadonlyArray): string | undefined => + editableKeys.length === 0 ? undefined - : planned.service === "database" && planned.paths.includes("config.version") - ? new StackCommandStartError({ - reason: "invalid-config", - message: "The requested database version does not match the saved stack binding", - suggestion: - "Keep the saved database version, or run supabase stack destroy to recreate the stack.", - }) - : new StackCommandStartError({ - reason: "invalid-config", - message: `The requested ${planned.service} ${planned.paths.join(", ")} cannot change on the saved stack`, - suggestion: - "Keep the saved endpoint and version settings, or run supabase stack destroy to recreate the stack.", - }); + : editableKeys.length === 1 + ? `Revert ${editableKeys[0]} to its saved value` + : "Revert the settings listed to their saved values"; + +/** Whether every one of these services' capabilities can be excluded with `--exclude`. */ +const allExcludable = (services: ReadonlyArray): boolean => + services.length > 0 && + services.every((service) => + (STACK_START_EXCLUDABLE_CAPABILITIES as ReadonlyArray).includes( + stackCapabilityForService(service), + ), + ); + +/** + * Grows a candidate `--exclude` set until requesting the remaining capabilities would actually + * start (validated with `studioNeedsRest`, the same check the real flag goes through), or gives + * up once a dependency can't itself be excluded. Bounded by the number of excludable capabilities, + * since each iteration adds at least one and today's only dependency (Studio needs REST) can't + * cycle back on itself. + */ +const resolveExclusionCapabilities = ( + creations: ReadonlyArray, + initialExclusions: ReadonlyArray, + candidateCapabilities: ReadonlyArray, +): ReadonlyArray | undefined => { + let capabilities = candidateCapabilities; + for (let attempt = 0; attempt <= STACK_START_EXCLUDABLE_CAPABILITIES.length; attempt++) { + const remaining = selectedCreations(creations, [...initialExclusions, ...capabilities]); + if (!studioNeedsRest(remaining)) return capabilities; + // Today's only capability dependency: Studio needs REST. Bail if Studio is already in the + // set (adding it again wouldn't help) or somehow isn't itself excludable. + if ( + capabilities.includes("studio") || + !(STACK_START_EXCLUDABLE_CAPABILITIES as ReadonlyArray).includes("studio") + ) + return undefined; + capabilities = dedupe([...capabilities, "studio"]); + } + return undefined; +}; + +/** Rejects every saved member whose endpoints or artifact versions the request would change. */ +const incompatibleChange = ( + planned: ReadonlyArray, + savedConfigById: ReadonlyMap, + requested: ReadonlyArray, + projectEnvValues: Readonly>, + stackIdentity: { readonly id: string; readonly name?: string }, + creations: ReadonlyArray, + currentExclusions: ReadonlyArray, +): + | { + readonly error: StackCommandStartError; + readonly changes: ReadonlyArray; + readonly command: string; + } + | undefined => { + const described = incompatibleSettingChanges( + planned, + savedConfigById, + requested, + projectEnvValues, + ); + if (described.length === 0) return undefined; + const changes = described.map(({ change }) => change); + const lines = settingChangeLines(changes); + const command = destroyCommandFor(stackIdentity.id); + const nameNote = stackIdentity.name === undefined ? "" : ` (stack ${stackIdentity.name})`; + const destroyClause = `\`${command}\`${nameNote} to recreate the stack — this permanently deletes its local database data.`; + + const settingKeys = dedupe( + described.filter((d) => d.reason === "setting").map((d) => d.change.key), + ); + const artifactKeys = dedupe( + described.filter((d) => d.reason === "artifact").map((d) => d.change.key), + ); + const pinnedKeys = dedupe( + described.filter((d) => d.reason === "pinned-artifact").map((d) => d.change.key), + ); + const endpointChanges = described.filter((d) => d.reason === "endpoint-presence"); + const addedEndpoints = endpointChanges.filter((d) => d.change.requested === "exposed"); + const removedEndpoints = endpointChanges.filter((d) => d.change.saved === "exposed"); + const addedEndpointKeys = dedupe(addedEndpoints.map((d) => d.change.key)); + const removedEndpointKeys = dedupe(removedEndpoints.map((d) => d.change.key)); + + const nonEditable = + artifactKeys.length > 0 || pinnedKeys.length > 0 || endpointChanges.length > 0; + const revert = revertAdvice(settingKeys); + + // Purely editable changes (ports, major_version): the original simple two-way choice. + if (!nonEditable && revert !== undefined) + return { + changes, + command, + error: new StackCommandStartError({ + reason: "invalid-config", + message: `The saved stack cannot adopt these changes: ${lines.join("; ")}`, + suggestion: `${revert} to keep the stack and its data, or run ${destroyClause}`, + }), + }; + + const sentences: Array = []; + if (artifactKeys.length > 0) + sentences.push( + `This CLI release starts a different ${artifactKeys.join(" and ")} than the saved stack.`, + ); + if (pinnedKeys.length > 0) + sentences.push( + `The saved stack pins an explicit ${pinnedKeys.join(" and ")} that this release would select by default anyway.`, + ); + if (addedEndpointKeys.length > 0) + sentences.push( + `This request needs the ${addedEndpointKeys.join(", ")} that the saved stack doesn't have.`, + ); + if (removedEndpointKeys.length > 0) + sentences.push( + `The saved stack has the ${removedEndpointKeys.join(", ")} that this request no longer needs.`, + ); + + // Requesting a new endpoint the saved stack never had is sometimes avoidable by excluding the + // whole capability instead of destroying the stack — but only when every incompatible change + // (not just the endpoint ones) belongs to a service this exclusion would drop from the request; + // an unrelated incompatibility elsewhere (e.g. a database major-version change) would still + // reject start regardless, so the exclusion would be offered without actually fixing anything. + const addedServices = dedupe(addedEndpoints.map((d) => d.change.service)) as ReadonlyArray< + PlannedInstance["service"] + >; + const excludableAddedServices = allExcludable(addedServices) + ? new Set(addedServices) + : undefined; + const canExcludeInstead = + excludableAddedServices !== undefined && + described.every((d) => excludableAddedServices.has(d.change.service)); + // A candidate exclusion that would itself fail to start (e.g. excluding REST while Studio stays + // requested) is resolved by pulling in a dependent capability if one is excludable, or dropped. + const resolvedCapabilities = canExcludeInstead + ? resolveExclusionCapabilities( + creations, + currentExclusions, + dedupe(addedServices.map(stackCapabilityForService)), + ) + : undefined; + const exclusionClause = + resolvedCapabilities !== undefined + ? ` Run the command again with \`--exclude ${resolvedCapabilities.join(",")}\` to open it without requesting ${resolvedCapabilities.length === 1 ? "that capability" : "those capabilities"}, or run ${destroyClause}` + : ` Run ${destroyClause}`; + + return { + changes, + command, + error: new StackCommandStartError({ + reason: "invalid-config", + message: `The saved stack cannot adopt these changes: ${lines.join("; ")}`, + suggestion: `${sentences.join(" ")}${exclusionClause}`, + }), + }; +}; const selectedCreations = ( creations: ReadonlyArray, @@ -451,10 +889,7 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags new StackCommandStartError({ reason: "invalid-config", message: cause.message, cause }), ), ); - if ( - requested.some(({ service }) => service === "studio") && - !requested.some(({ service }) => service === "rest") - ) + if (studioNeedsRest(requested)) return yield* new StackCommandStartError({ reason: "flags", message: "Studio cannot be started without the REST API capability", @@ -517,9 +952,28 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags const initialComposition = composition.members.length === 0; const serviceKindsChanged = !sameKinds(currentInstances, requested); const planned = yield* stack.composition.plan(requested).pipe(Effect.mapError(stackError)); - for (const entry of planned) { - const rejected = entry.member ? incompatibleChange(entry) : undefined; - if (rejected !== undefined) return yield* rejected; + const stackIdentity = { + id: stack.id, + ...(target.name === undefined ? {} : { name: target.name }), + }; + const savedConfigById = new Map(currentStatuses.map(({ id, config: saved }) => [id, saved])); + const rejected = incompatibleChange( + planned, + savedConfigById, + requested, + config.projectEnvValues, + stackIdentity, + creations, + exclusions, + ); + if (rejected !== undefined) { + const machineErrorContext = yield* Effect.serviceOption(MachineErrorContext); + if (Option.isSome(machineErrorContext)) + yield* machineErrorContext.value.set({ + stack_changes: rejected.changes, + recreate_command: rejected.command, + }); + return yield* rejected.error; } const reuseIds: Array = planned.filter(({ member }) => member).map(({ id }) => id); for (const creation of requested) { diff --git a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts index 8c794ae654..831e6787eb 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts @@ -4,12 +4,14 @@ import { describe, expect, it } from "@effect/vitest"; import { Deferred, Effect, - Equal, Fiber, FileSystem, Layer, Option, Redacted, + Schema, + Sink, + Stdio, Stream, } from "effect"; import { @@ -17,23 +19,28 @@ import { DEFAULT_LOCAL_JWT_SECRET, DEFAULT_POSTGRES_ROOT_KEY, } from "@supabase/stack/defaults"; -import { postgresVersion } from "@supabase/stack/internal/artifacts"; +import { catalogPins, postgresVersion } from "@supabase/stack/internal/artifacts"; import { StackError, type ServiceCreation, type ServiceCreationInput, - type PlannedInstance, type ServiceInstance, type ServiceInstances, type StackCredentials, type Stack, } from "@supabase/stack/effect"; +import { planSupabaseComposition } from "@supabase/stack/internal/composition"; import { mockCommandSettings, mockTelemetryStateTracked, withEnvVar, } from "../../../../../tests/helpers/command-mocks.ts"; -import { mockOutput, mockRuntimeInfo, mockTty } from "../../../../../tests/helpers/mocks.ts"; +import { + mockOutput, + mockProcessControl, + mockRuntimeInfo, + mockTty, +} from "../../../../../tests/helpers/mocks.ts"; import { DbConnection, type DbSession, @@ -44,6 +51,9 @@ import { CommandPlatformApiFactory } from "../../../../auth/command-platform-api import { stdinLayer } from "../../../../shared/runtime/stdin.layer.ts"; import * as HttpClient from "effect/unstable/http/HttpClient"; import { CliArgs } from "../../../../shared/cli/cli-args.service.ts"; +import { withJsonErrorHandling } from "../../../../shared/output/json-error-handling.ts"; +import { machineErrorContextLayer } from "../../../../shared/output/machine-error-context.layer.ts"; +import { jsonOutputLayer, streamJsonOutputLayer } from "../../../../shared/output/output.layer.ts"; import { StackApi, stackApiLayer, StackTargetResolver } from "../stack.shared.ts"; import { stackStart } from "./start.handler.ts"; import { StackCommandStartError } from "./start.errors.ts"; @@ -129,11 +139,13 @@ const instance = ( service: "database", restart: (input?: Parameters[0]) => Effect.sync(() => { - if (input?.config.version !== undefined) - current = { - ...current, - config: { ...current.config, version: input.config.version }, - }; + if (input === undefined) return; + current = { + ...current, + ...("version" in input ? { version: input.version } : {}), + ...("endpoints" in input ? { endpoints: input.endpoints } : {}), + config: { ...current.config, ...input.config }, + }; }), status: Effect.sync(() => status(current)), credentials: () => @@ -143,8 +155,18 @@ const instance = ( resetData: Effect.die("unused"), }; } - case "rest": - return { ...base, service: "rest" }; + case "rest": { + let current = creation; + return { + ...base, + service: "rest", + restart: (input?: Parameters[0]) => + Effect.sync(() => { + if (input !== undefined) current = { ...current, ...input }; + }), + status: Effect.sync(() => status(current)), + }; + } case "auth": return { ...base, service: "auth" }; case "realtime": @@ -169,8 +191,18 @@ const instance = ( return { ...base, service: "studio" }; case "pgmeta": return { ...base, service: "pgmeta" }; - case "mail": - return { ...base, service: "mail" }; + case "mail": { + let current = creation; + return { + ...base, + service: "mail", + restart: (input?: Parameters[0]) => + Effect.sync(() => { + if (input !== undefined) current = { ...current, ...input }; + }), + status: Effect.sync(() => status(current)), + }; + } case "analytics": return { ...base, service: "analytics" }; case "vector": @@ -304,29 +336,28 @@ const fakeStack = (compositionStart?: Stack["composition"]["start"]) => { ); return members; }), - plan: (creations) => + // Delegates to the production planner so paths/shared-API-port normalization match what + // `packages/stack` actually reports, instead of a hand-rolled approximation. + plan: (creations: ReadonlyArray) => Effect.forEach(members, (member) => - member.status.pipe( - Effect.map(({ config }): ReadonlyArray => { - const request = creations.find(({ service }) => service === member.service); - if (request === undefined) return []; - const base = { id: member.id, service: member.service, member: true }; - if (config.service === "database" && request.service === "database") - return [ - postgresVersion(config.config.version) === postgresVersion(request.config.version) - ? { ...base, change: "unchanged" } - : { ...base, change: "incompatible", paths: ["config.version"] }, - ]; - if (!Equal.equals(config.endpoints, request.endpoints)) - return [{ ...base, change: "incompatible", paths: ["endpoints"] }]; - return [ - Equal.equals(config.config, request.config) - ? { ...base, change: "unchanged" } - : { ...base, change: "changed", paths: ["config"] }, - ]; - }), + member.status.pipe(Effect.map(({ config }) => ({ id: member.id, creation: config }))), + ).pipe( + Effect.map((instances) => + planSupabaseComposition( + { + instances, + composition: { + members: members.map(({ id }) => ({ + id, + activation: activations.get(id) ?? "eager", + })), + dependencies: [], + }, + }, + creations, + ), ), - ).pipe(Effect.map((planned) => planned.flat())), + ), configure: ({ members: configured }) => Effect.sync(() => { activations = new Map(configured.map(({ id, activation }) => [id, activation])); @@ -397,7 +428,7 @@ const fakeStack = (compositionStart?: Stack["composition"]["start"]) => { const layers = ( root: string, fixture: ReturnType, - output = mockOutput(), + output: Pick, "layer"> = mockOutput(), existing = true, explicitWorkdir = false, // Fixtures request the native runtime, so pin a host that ships native artifacts. @@ -444,6 +475,57 @@ const layers = ( ); }; +const machineEnvelope = Schema.decodeEffect( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)), +); + +/** + * A real captured `Stdio` layer, needed only by the JSON/stream-json failure-envelope tests below + * since `machineErrorContextLayer`'s merge into the error envelope lives inside the real + * `jsonOutputLayer`/`streamJsonOutputLayer` `fail` implementations, which `mockOutput()` never + * replicates. + */ +const mockCapturingStdio = () => { + const stdout: Array = []; + const layer = Layer.succeed( + Stdio.Stdio, + Stdio.make({ + args: Effect.succeed([]), + stdin: Stream.empty, + stdout: () => + Sink.forEach((item: string | Uint8Array) => + Effect.sync(() => { + stdout.push(typeof item === "string" ? item : new TextDecoder().decode(item)); + }), + ), + stderr: () => Sink.forEach(() => Effect.void), + }), + ); + return { layer, stdout }; +}; + +/** + * Wires the real `jsonOutputLayer`/`streamJsonOutputLayer` over a captured `Stdio`, with + * `machineErrorContextLayer` merged alongside it (matching `start.command.ts`'s composition) so + * the handler and the output layer's `fail` share the same live cell, plus a real + * `mockProcessControl()` since `withJsonErrorHandling` sets the exit code on it. + */ +const jsonErrorLayers = ( + root: string, + fixture: ReturnType, + format: "json" | "stream-json", +) => { + const stdio = mockCapturingStdio(); + const processControl = mockProcessControl(); + const outputLayer = format === "json" ? jsonOutputLayer : streamJsonOutputLayer; + const layer = Layer.mergeAll( + layers(root, fixture, { layer: outputLayer.pipe(Layer.provide(stdio.layer)) }), + machineErrorContextLayer, + processControl.layer, + ); + return { layer, stdio, processControl }; +}; + describe("experimental stack start", () => { it.live("rejects incompatible Functions env before changing composition", () => Effect.gen(function* () { @@ -1152,18 +1234,21 @@ describe("experimental stack start", () => { }).pipe(Effect.provide(BunServices.layer)), ); - it.live("rejects a changed endpoint after the stack is stopped", () => + it.live("names the config key and both values when a saved port changes", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-changed-port-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-port-config-" }); yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "changed-port"\n'); + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "port-config"\n[api]\nport = 54321\n', + ); const fixture = fakeStack(); yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); yield* fs.writeFileString( `${root}/supabase/config.toml`, - 'project_id = "changed-port"\n[api]\nport = 54999\n', + 'project_id = "port-config"\n[api]\nport = 54999\n', ); yield* fixture.stack.composition.stop; const error = yield* stackStart(flags()).pipe( @@ -1173,13 +1258,759 @@ describe("experimental stack start", () => { expect(error).toMatchObject({ reason: "invalid-config", - message: expect.stringContaining("cannot change on the saved stack"), - suggestion: expect.stringContaining("supabase stack destroy"), + message: expect.stringContaining("[api] port: saved 54321, requested 54999"), }); + // The rejection leaves the stopped composition untouched: no recompose was attempted. expect(fixture.composed).toBe(1); }).pipe(Effect.provide(BunServices.layer)), ); + it.live("names a dedicated (non-shared) port's own config key", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-dedicated-port-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "dedicated-port"\n'); + const fixture = fakeStack(); + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "dedicated-port"\n[studio]\nport = 12345\n', + ); + yield* fixture.stack.composition.stop; + const error = yield* stackStart(flags()).pipe( + Effect.provide(layers(root, fixture)), + Effect.flip, + ); + + expect(error).toMatchObject({ + reason: "invalid-config", + message: expect.stringContaining("[studio] port: saved automatic, requested 12345"), + }); + // A dedicated port only affects its own service, unlike the shared API port. + expect(error).toBeInstanceOf(StackCommandStartError); + if (error instanceof StackCommandStartError) + expect(error.message).not.toContain("[api] port"); + }).pipe(Effect.provide(BunServices.layer)), + ); + + // The production planner (`fixedApiPorts`/`withSharedApiPort`) normalizes a requested + // automatic shared-API port to the composition's already-fixed value whenever one exists, so a + // saved fixed port going back to automatic in `config.toml` reuses the saved port rather than + // failing. This locks down that non-obvious compatible case: it is not an incompatible path. + it.live( + "accepts a shared API port going from fixed back to automatic, reusing the saved port", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-api-to-auto-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "api-to-auto"\n[api]\nport = 54321\n', + ); + const fixture = fakeStack(); + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + + yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "api-to-auto"\n'); + yield* fixture.stack.composition.stop; + // Does not throw: the planner treats this as compatible (`change: "unchanged"`), not an + // incompatible path to report. Reusing the already-bound port for the resumed instance is + // `packages/stack`'s own concern, not asserted here. + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live( + "collects simultaneous database-version and port changes into one error with plural revert wording", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-multi-change-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "multi-change"\n'); + const fixture = fakeStack(); + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "multi-change"\n[db]\nmajor_version = 15\n[studio]\nport = 12345\n', + ); + yield* fixture.stack.composition.stop; + const error = yield* stackStart(flags()).pipe( + Effect.provide(layers(root, fixture)), + Effect.flip, + ); + + expect(error).toMatchObject({ + reason: "invalid-config", + message: expect.stringContaining("[db] major_version: saved 17, requested 15"), + }); + expect(error).toMatchObject({ + message: expect.stringContaining("[studio] port: saved automatic, requested 12345"), + }); + expect(error).toBeInstanceOf(StackCommandStartError); + if (error instanceof StackCommandStartError) + expect(error.suggestion).toContain( + "Revert the settings listed to their saved values to keep the stack and its data", + ); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live("names the env var override when SUPABASE_*_PORT set the saved port", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-port-env-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "port-env"\n'); + const fixture = fakeStack(); + yield* withEnvVar( + "SUPABASE_API_PORT", + "54321", + stackStart(flags()).pipe(Effect.provide(layers(root, fixture))), + ); + + yield* fixture.stack.composition.stop; + const error = yield* withEnvVar( + "SUPABASE_API_PORT", + "54999", + stackStart(flags()).pipe(Effect.provide(layers(root, fixture)), Effect.flip), + ); + + expect(error).toMatchObject({ + reason: "invalid-config", + message: expect.stringContaining("SUPABASE_API_PORT: saved 54321, requested 54999"), + }); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live( + "names [db] major_version and the destroy command when the saved Postgres version changes", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-major-version-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "major-version"\n'); + const fixture = fakeStack(); + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "major-version"\n[db]\nmajor_version = 15\n', + ); + yield* fixture.stack.composition.stop; + const error = yield* stackStart(flags()).pipe( + Effect.provide(layers(root, fixture)), + Effect.flip, + ); + + expect(error).toMatchObject({ + reason: "invalid-config", + message: expect.stringContaining("[db] major_version: saved 17, requested 15"), + suggestion: expect.stringContaining( + `Revert [db] major_version to its saved value to keep the stack and its data, or run \`supabase stack destroy --stack-id ${fixture.stack.id}\` to recreate the stack`, + ), + }); + expect(error).toMatchObject({ suggestion: expect.stringContaining("database data") }); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live( + "names the full Postgres build, not major_version, when only the pinned build differs", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-pg-build-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "pg-build"\n[db]\nmajor_version = 17\n', + ); + const fixture = fakeStack(); + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + const database = fixture.members.find(({ service }) => service === "database"); + if (database?.service !== "database") return yield* Effect.die("Database missing"); + const observed = yield* database.status; + if (observed.config.service !== "database") + return yield* Effect.die("Database config missing"); + const pinnedVersion = postgresVersion("17"); + // A saved build the current catalog no longer pins (`postgresVersion` only normalizes a + // recognized alias): same major as the requested `17`, different full build. + yield* database.restart({ + config: { ...observed.config.config, version: "17.0.0-stale-build" }, + }); + yield* fixture.stack.composition.stop; + const error = yield* stackStart(flags()).pipe( + Effect.provide(layers(root, fixture)), + Effect.flip, + ); + + expect(error).toMatchObject({ + reason: "invalid-config", + message: expect.stringContaining( + `Postgres build: saved 17.0.0-stale-build, requested ${pinnedVersion}`, + ), + }); + expect(error).toBeInstanceOf(StackCommandStartError); + if (error instanceof StackCommandStartError) { + expect(error.message).not.toContain("major_version"); + expect(error.suggestion).not.toContain("Revert"); + expect(error.suggestion).toContain( + "This CLI release starts a different Postgres build than the saved stack.", + ); + expect(error.suggestion).toContain( + `supabase stack destroy --stack-id ${fixture.stack.id}`, + ); + } + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live( + "drops the revert sentence for an artifact-version-only mismatch and explains the fix in plain language", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-artifact-version-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "artifact-version"\n', + ); + const fixture = fakeStack(); + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + const rest = fixture.members.find(({ service }) => service === "rest"); + if (rest?.service !== "rest") return yield* Effect.die("REST missing"); + const restObserved = yield* rest.status; + if (restObserved.config.service !== "rest") return yield* Effect.die("REST config missing"); + yield* rest.restart({ ...restObserved.config, version: "rest-v1-stale" }); + yield* fixture.stack.composition.stop; + const error = yield* stackStart(flags()).pipe( + Effect.provide(layers(root, fixture)), + Effect.flip, + ); + + expect(error).toMatchObject({ + reason: "invalid-config", + message: expect.stringContaining("rest artifact version"), + }); + expect(error).toBeInstanceOf(StackCommandStartError); + if (error instanceof StackCommandStartError) { + expect(error.suggestion).not.toContain("Revert"); + expect(error.suggestion).toContain( + "This CLI release starts a different rest artifact version than the saved stack.", + ); + expect(error.suggestion).toContain( + `supabase stack destroy --stack-id ${fixture.stack.id}`, + ); + } + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live( + "uses the destroy-only wording when a non-editable change accompanies an editable one", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-mixed-editable-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "mixed-editable"\n', + ); + const fixture = fakeStack(); + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + const rest = fixture.members.find(({ service }) => service === "rest"); + if (rest?.service !== "rest") return yield* Effect.die("REST missing"); + const restObserved = yield* rest.status; + if (restObserved.config.service !== "rest") return yield* Effect.die("REST config missing"); + yield* rest.restart({ ...restObserved.config, version: "rest-v1-stale" }); + yield* fixture.stack.composition.stop; + + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "mixed-editable"\n[studio]\nport = 12345\n', + ); + const error = yield* stackStart(flags()).pipe( + Effect.provide(layers(root, fixture)), + Effect.flip, + ); + + expect(error).toMatchObject({ + reason: "invalid-config", + // The editable change still appears in the message even though reverting it alone + // can't unblock start: the non-editable artifact-version change still would. + message: expect.stringContaining("[studio] port: saved automatic, requested 12345"), + }); + expect(error).toBeInstanceOf(StackCommandStartError); + if (error instanceof StackCommandStartError) { + expect(error.suggestion).not.toContain("Revert"); + expect(error.suggestion).toContain( + "This CLI release starts a different rest artifact version than the saved stack.", + ); + expect(error.suggestion).toContain( + `supabase stack destroy --stack-id ${fixture.stack.id}`, + ); + } + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live("emits structured stack_changes and recreate_command on the JSON error envelope", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-json-envelope-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "json-envelope"\n'); + const fixture = fakeStack(); + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "json-envelope"\n[db]\nmajor_version = 15\n', + ); + yield* fixture.stack.composition.stop; + const { layer, stdio } = jsonErrorLayers(root, fixture, "json"); + yield* stackStart(flags()).pipe(withJsonErrorHandling, Effect.provide(layer)); + + expect(stdio.stdout).toHaveLength(1); + const envelope = yield* machineEnvelope(stdio.stdout[0]!); + expect(envelope._tag).toBe("Error"); + expect(envelope.error).toMatchObject({ + code: "ExperimentalStackStartError", + message: expect.stringContaining("[db] major_version: saved 17, requested 15"), + }); + expect(envelope.stack_changes).toEqual([ + { + service: "database", + path: "config.version", + key: "[db] major_version", + saved: "17", + requested: "15", + editable: true, + }, + ]); + expect(envelope.recreate_command).toBe( + `supabase stack destroy --stack-id ${fixture.stack.id}`, + ); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live( + "reports one stack_changes entry per affected service while collapsing the text to one line", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-start-shared-port-dedup-", + }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "shared-port-dedup"\n[api]\nport = 54321\n', + ); + const fixture = fakeStack(); + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + // The shared API port is bound by every API-backed service the default config enables; + // this only exercises the dedup/per-entry contract if more than one of them is present. + const apiBackedServices = fixture.members + .map(({ service }) => service) + .filter((service) => + ["rest", "auth", "realtime", "storage", "functions"].includes(service), + ); + expect(apiBackedServices.length).toBeGreaterThan(1); + + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "shared-port-dedup"\n[api]\nport = 54999\n', + ); + yield* fixture.stack.composition.stop; + const { layer, stdio } = jsonErrorLayers(root, fixture, "json"); + yield* stackStart(flags()).pipe(withJsonErrorHandling, Effect.provide(layer)); + + const envelope = yield* machineEnvelope(stdio.stdout[0]!); + expect(envelope.error).toMatchObject({ + // The deduplicated text line appears exactly once, not once per affected service. + message: `The saved stack cannot adopt these changes: [api] port: saved 54321, requested 54999`, + }); + const changes = envelope.stack_changes as ReadonlyArray<{ + readonly service: string; + readonly path: string; + readonly key: string; + }>; + expect(changes).toHaveLength(apiBackedServices.length); + expect(new Set(changes.map((change) => change.service))).toEqual( + new Set(apiBackedServices), + ); + for (const change of changes) + expect(change).toMatchObject({ path: "endpoints.http.port", key: "[api] port" }); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live("emits the same structured error fields on the stream-json terminal event", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-stream-json-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "stream-json"\n'); + const fixture = fakeStack(); + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "stream-json"\n[db]\nmajor_version = 15\n', + ); + yield* fixture.stack.composition.stop; + const { layer, stdio } = jsonErrorLayers(root, fixture, "stream-json"); + yield* stackStart(flags()).pipe(withJsonErrorHandling, Effect.provide(layer)); + + const event = yield* machineEnvelope(stdio.stdout.at(-1)!); + expect(event.type).toBe("error"); + expect(event.error).toMatchObject({ + code: "ExperimentalStackStartError", + message: expect.stringContaining("[db] major_version: saved 17, requested 15"), + }); + expect(event.stack_changes).toEqual([ + { + service: "database", + path: "config.version", + key: "[db] major_version", + saved: "17", + requested: "15", + editable: true, + }, + ]); + expect(event.recreate_command).toBe(`supabase stack destroy --stack-id ${fixture.stack.id}`); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live("always suggests --stack-id for a named stack, naming the stack as plain text", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-named-destroy-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "named-destroy"\n'); + const fixture = fakeStack(); + const target = Layer.succeed(StackTargetResolver, { + resolve: () => + Effect.succeed({ + projectRoot: root, + id: fixture.stack.id, + name: "feature-a", + runtime: "native" as const, + hostRunning: false, + }), + }); + yield* stackStart(flags()).pipe( + Effect.provide(Layer.mergeAll(layers(root, fixture), target)), + ); + + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "named-destroy"\n[db]\nmajor_version = 15\n', + ); + yield* fixture.stack.composition.stop; + const error = yield* stackStart(flags()).pipe( + Effect.provide(Layer.mergeAll(layers(root, fixture), target)), + Effect.flip, + ); + + expect(error).toMatchObject({ + suggestion: expect.stringContaining( + `supabase stack destroy --stack-id ${fixture.stack.id}\` (stack feature-a)`, + ), + }); + expect(error).toBeInstanceOf(StackCommandStartError); + if (error instanceof StackCommandStartError) + expect(error.suggestion).not.toContain("--stack feature-a"); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live("names the endpoint a database-only saved stack gains when reopened by the CLI", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-database-only-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "database-only"\n'); + const fixture = fakeStack(); + const excluded = [ + "rest", + "auth", + "realtime", + "storage", + "functions", + "studio", + "mail", + "analytics", + "pooler", + ]; + yield* stackStart(flags(excluded)).pipe(Effect.provide(layers(root, fixture))); + expect(fixture.members.map(({ service }) => service)).toEqual(["database"]); + + const database = fixture.members.find(({ service }) => service === "database"); + if (database?.service !== "database") return yield* Effect.die("Database missing"); + const observed = yield* database.status; + if (observed.config.service !== "database") + return yield* Effect.die("Database config missing"); + // A saved composition created with no public endpoint at all — the CLI's own requests + // always include `endpoints.sql`, but a stack created directly through the package + // (bypassing the CLI) can save a database member without one. + yield* database.restart({ ...observed.config, endpoints: undefined }); + yield* fixture.stack.composition.stop; + + const error = yield* stackStart(flags(excluded)).pipe( + Effect.provide(layers(root, fixture)), + Effect.flip, + ); + + expect(error).toMatchObject({ + reason: "invalid-config", + message: expect.stringContaining( + "database sql endpoint: saved not exposed, requested exposed", + ), + }); + expect(error).toBeInstanceOf(StackCommandStartError); + if (error instanceof StackCommandStartError) { + // This isn't an artifact/version difference, so it must not borrow that wording. + expect(error.suggestion).not.toContain("This CLI release starts a different"); + expect(error.suggestion).toContain( + "This request needs the database sql endpoint that the saved stack doesn't have.", + ); + // The database capability can't be excluded, so destroying is the only offered fix. + expect(error.suggestion).not.toContain("--exclude"); + expect(error.suggestion).toContain(`supabase stack destroy --stack-id ${fixture.stack.id}`); + } + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live( + "names each endpoint a saved Mail service gains when reopened by the CLI, excluding mail to fix it", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-mail-presence-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "mail-presence"\n'); + const fixture = fakeStack(); + const excluded = [ + "rest", + "auth", + "realtime", + "storage", + "functions", + "studio", + "analytics", + "pooler", + ]; + yield* stackStart(flags(excluded)).pipe(Effect.provide(layers(root, fixture))); + expect(fixture.members.map(({ service }) => service).toSorted()).toEqual([ + "database", + "mail", + ]); + + const mail = fixture.members.find(({ service }) => service === "mail"); + if (mail?.service !== "mail") return yield* Effect.die("Mail missing"); + // A saved Mail service exposing only `smtp` — a composition the package itself can create + // directly (see `Supabase.native.integration.test.ts`), unlike the CLI, which always + // requests all three Mail endpoints. + yield* mail.restart({ service: "mail", config: {}, endpoints: { smtp: { port: "auto" } } }); + yield* fixture.stack.composition.stop; + + const error = yield* stackStart(flags(excluded)).pipe( + Effect.provide(layers(root, fixture)), + Effect.flip, + ); + + expect(error).toMatchObject({ + reason: "invalid-config", + message: expect.stringContaining( + "mail http endpoint: saved not exposed, requested exposed", + ), + }); + expect(error).toMatchObject({ + message: expect.stringContaining( + "mail pop3 endpoint: saved not exposed, requested exposed", + ), + }); + expect(error).toBeInstanceOf(StackCommandStartError); + if (error instanceof StackCommandStartError) { + // A per-endpoint presence change: not an editable port setting, so it must not borrow + // that wording or a config key that implies the user could just set a port. + expect(error.message).not.toContain("saved unset, requested automatic"); + expect(error.message).not.toContain("[local_smtp]"); + expect(error.suggestion).not.toContain("Revert"); + expect(error.suggestion).toContain( + "This request needs the mail http endpoint, mail pop3 endpoint that the saved stack doesn't have.", + ); + // Every incompatible change here belongs to Mail, so excluding it is offered. + expect(error.suggestion).toContain("--exclude mail"); + expect(error.suggestion).toContain( + `supabase stack destroy --stack-id ${fixture.stack.id}`, + ); + } + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live( + "does not offer --exclude when an unrelated incompatible change would still reject start", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-mail-plus-major-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "mail-plus-major"\n', + ); + const fixture = fakeStack(); + const excluded = [ + "rest", + "auth", + "realtime", + "storage", + "functions", + "studio", + "analytics", + "pooler", + ]; + yield* stackStart(flags(excluded)).pipe(Effect.provide(layers(root, fixture))); + + const mail = fixture.members.find(({ service }) => service === "mail"); + if (mail?.service !== "mail") return yield* Effect.die("Mail missing"); + // Same database-plus-Mail saved stack as above, but this time the config also moves the + // database to a different major version, an incompatibility `--exclude mail` cannot fix. + yield* mail.restart({ service: "mail", config: {}, endpoints: { smtp: { port: "auto" } } }); + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "mail-plus-major"\n[db]\nmajor_version = 15\n', + ); + yield* fixture.stack.composition.stop; + + const error = yield* stackStart(flags(excluded)).pipe( + Effect.provide(layers(root, fixture)), + Effect.flip, + ); + + expect(error).toMatchObject({ + reason: "invalid-config", + message: expect.stringContaining("[db] major_version: saved 17, requested 15"), + }); + expect(error).toMatchObject({ + message: expect.stringContaining( + "mail http endpoint: saved not exposed, requested exposed", + ), + }); + expect(error).toBeInstanceOf(StackCommandStartError); + if (error instanceof StackCommandStartError) { + // `--exclude mail` alone wouldn't fix the database major-version mismatch, so it must + // not be offered as if it were sufficient. + expect(error.suggestion).not.toContain("--exclude"); + expect(error.suggestion).toContain( + `supabase stack destroy --stack-id ${fixture.stack.id}`, + ); + } + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live( + "pulls Studio into the suggested --exclude set when excluding REST alone would strand it", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-start-exclude-dependency-", + }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "exclude-dependency"\n', + ); + const fixture = fakeStack(); + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + + const rest = fixture.members.find(({ service }) => service === "rest"); + if (rest?.service !== "rest") return yield* Effect.die("REST missing"); + const restObserved = yield* rest.status; + if (restObserved.config.service !== "rest") return yield* Effect.die("REST config missing"); + // A saved REST service with its HTTP endpoint omitted — Auth stays exposed and Studio + // stays enabled exactly as the CLI would request them, so a bare `--exclude rest` would + // look sufficient but leaves Studio without the REST API it depends on. + yield* rest.restart({ ...restObserved.config, endpoints: {} }); + yield* fixture.stack.composition.stop; + + const error = yield* stackStart(flags()).pipe( + Effect.provide(layers(root, fixture)), + Effect.flip, + ); + + expect(error).toMatchObject({ + reason: "invalid-config", + message: expect.stringContaining( + "rest http endpoint: saved not exposed, requested exposed", + ), + }); + expect(error).toBeInstanceOf(StackCommandStartError); + if (error instanceof StackCommandStartError) { + // `--exclude rest` alone would itself fail the Studio-without-REST guard, so Studio + // must be folded into the suggested exclusion instead of offered as a broken command. + expect(error.suggestion).toContain("--exclude rest,studio"); + expect(error.suggestion).not.toContain("`--exclude rest`"); + expect(error.suggestion).toContain( + `supabase stack destroy --stack-id ${fixture.stack.id}`, + ); + } + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live( + "treats an explicit saved artifact version equal to the catalog default as a formality", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-pinned-default-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "pinned-default"\n', + ); + const fixture = fakeStack(); + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + const rest = fixture.members.find(({ service }) => service === "rest"); + if (rest?.service !== "rest") return yield* Effect.die("REST missing"); + const restObserved = yield* rest.status; + if (restObserved.config.service !== "rest") return yield* Effect.die("REST config missing"); + const restDefaultVersion = catalogPins().find( + (pin) => pin.service === "rest" && pin.isDefault, + )?.pin.upstreamVersion; + if (restDefaultVersion === undefined) return yield* Effect.die("No default rest pin"); + // Pins the exact version the catalog would pick anyway: a formal difference only, since + // the CLI's own requests never set a top-level `version` and so always fall back to it. + yield* rest.restart({ ...restObserved.config, version: restDefaultVersion }); + yield* fixture.stack.composition.stop; + const error = yield* stackStart(flags()).pipe( + Effect.provide(layers(root, fixture)), + Effect.flip, + ); + + expect(error).toMatchObject({ + reason: "invalid-config", + message: expect.stringContaining( + `rest artifact version: saved ${restDefaultVersion}, requested ${restDefaultVersion}`, + ), + }); + expect(error).toBeInstanceOf(StackCommandStartError); + if (error instanceof StackCommandStartError) { + expect(error.suggestion).not.toContain("This CLI release starts a different"); + expect(error.suggestion).toContain( + "The saved stack pins an explicit rest artifact version that this release would select by default anyway.", + ); + expect(error.suggestion).toContain( + `supabase stack destroy --stack-id ${fixture.stack.id}`, + ); + } + }).pipe(Effect.provide(BunServices.layer)), + ); + it.live("matches a Postgres major alias to the saved pinned database version", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/packages/stack/package.json b/packages/stack/package.json index 69afe969c5..a5bd12f737 100644 --- a/packages/stack/package.json +++ b/packages/stack/package.json @@ -11,6 +11,7 @@ "./commands": "./src/Commands.ts", "./internal/dispatch": "./src/internal/dispatch.ts", "./internal/artifacts": "./src/internal/artifacts.ts", + "./internal/composition": "./src/internal/composition.ts", "./internal/release": "./src/internal/release.ts", "./internal/test-run-label": "./src/internal/test-run-label.ts" }, diff --git a/packages/stack/src/internal/composition.ts b/packages/stack/src/internal/composition.ts new file mode 100644 index 0000000000..830238b291 --- /dev/null +++ b/packages/stack/src/internal/composition.ts @@ -0,0 +1,2 @@ +/** Composition planner shared with the CLI's test fakes to reproduce saved-stack comparison. */ +export { planSupabaseComposition } from "../composition/Supabase.ts"; From 9d79b0915ec439c459b9224ee6cabeed57064956 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sun, 4 Oct 2026 13:53:59 +0200 Subject: [PATCH 024/142] refactor(stack): own launch scopes before preparation, add gateway drain operations --- .oxlintrc.json | 2 + .../stack/src/HttpProxy.integration.test.ts | 189 ++++++++++++- packages/stack/src/HttpProxy.ts | 34 ++- packages/stack/src/Network.ts | 3 - packages/stack/src/Owner.integration.test.ts | 220 ++++++++++++++- packages/stack/src/Owner.ts | 48 +++- packages/stack/src/Proxy.integration.test.ts | 250 +++++++++++++++++- packages/stack/src/Proxy.ts | 167 ++++++++++-- .../stack/src/Service.integration.test.ts | 93 +++++++ packages/stack/src/Service.ts | 160 ++++++++--- .../DockerDatabaseStorage.integration.test.ts | 171 ++++++++++++ .../src/storage/DockerDatabaseStorage.ts | 25 +- 12 files changed, 1272 insertions(+), 90 deletions(-) diff --git a/.oxlintrc.json b/.oxlintrc.json index 10f646e45d..dee35ff5dc 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -80,6 +80,7 @@ "packages/stack/src/Owner.ts", "packages/stack/src/Ports.ts", "packages/stack/src/Ports.integration.test.ts", + "packages/stack/src/Owner.integration.test.ts", "packages/stack/src/port-reservation.e2e.test.ts", "packages/stack/src/StackHost.ts", "packages/stack/src/host/CommandRunner.ts", @@ -266,6 +267,7 @@ "packages/stack/src/Owner.ts", "packages/stack/src/Ports.ts", "packages/stack/src/Ports.integration.test.ts", + "packages/stack/src/Owner.integration.test.ts", "packages/stack/src/port-reservation.e2e.test.ts", "packages/stack/src/StackHost.ts", "packages/stack/src/preparation/ArtifactStore.ts", diff --git a/packages/stack/src/HttpProxy.integration.test.ts b/packages/stack/src/HttpProxy.integration.test.ts index 8e6398d412..112ecbdeea 100644 --- a/packages/stack/src/HttpProxy.integration.test.ts +++ b/packages/stack/src/HttpProxy.integration.test.ts @@ -1,6 +1,6 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Data, Deferred, Effect, Fiber, Layer } from "effect"; +import { Data, Deferred, Effect, Fiber, Layer, Stream, SubscriptionRef } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { createServer, type Server, type ServerResponse } from "node:http"; // oxlint-disable-line effecttsgo/node-builtin-import -- raw server fixture. import { createServer as createTcpServer, Socket, type Server as NetServer } from "node:net"; // oxlint-disable-line effecttsgo/node-builtin-import -- raw disconnect fixture. @@ -1027,3 +1027,190 @@ it.live("releases a waiting WebSocket target quietly when its client resets", () }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, captureErrors(logs)))); }); + +const connectRaw = (port: number, host: string) => + Effect.acquireRelease( + Effect.callback((resume) => { + const socket = new Socket(); + const onConnect = () => resume(Effect.succeed(socket)); + const onError = (cause: Error) => + resume(Effect.fail(new HttpProxyTestError({ message: cause.message, cause }))); + socket.once("connect", onConnect); + socket.once("error", onError); + socket.connect(port, host); + return Effect.sync(() => { + socket.off("connect", onConnect); + socket.off("error", onError); + }); + }).pipe(Effect.timeout("5 seconds")), + (socket) => Effect.sync(() => socket.destroy()), + ); + +/** Reads one full HTTP/1.1 response off `socket`: accumulates chunks until the declared body length. */ +const readFullResponse = (socket: Socket) => + Effect.callback<{ readonly status: number; readonly body: string }, HttpProxyTestError>( + (resume) => { + let buffered = Buffer.alloc(0); + let headerEnd: number | undefined; + let status = 0; + let contentLength = 0; + const onData = (chunk: Buffer) => { + buffered = Buffer.concat([buffered, chunk]); + if (headerEnd === undefined) { + const index = buffered.indexOf("\r\n\r\n"); + if (index === -1) return; + headerEnd = index + 4; + const head = buffered.subarray(0, headerEnd).toString("latin1"); + status = Number(/^HTTP\/1\.1 (\d+)/u.exec(head)?.[1] ?? 0); + contentLength = Number(/content-length:\s*(\d+)/iu.exec(head)?.[1] ?? 0); + } + if (buffered.length - headerEnd < contentLength) return; + socket.off("data", onData); + resume( + Effect.succeed({ + status, + body: buffered.subarray(headerEnd, headerEnd + contentLength).toString(), + }), + ); + }; + socket.on("data", onData); + return Effect.sync(() => socket.off("data", onData)); + }, + ).pipe(Effect.timeout("5 seconds")); + +/** + * Resolves once a fresh connection attempt to `port` is refused rather than served: the listener + * accepts the TCP handshake but destroys the socket immediately, so no response ever arrives. + */ +const connectionRefused = (port: number, host: string) => + Effect.acquireRelease( + Effect.sync(() => new Socket()), + (socket) => Effect.sync(() => socket.destroy()), + ).pipe( + Effect.flatMap((probe) => + Effect.callback((resume) => { + const onSettled = (responded: boolean) => resume(Effect.succeed(!responded)); + probe.once("error", () => onSettled(false)); + probe.once("close", () => onSettled(false)); + probe.once("data", () => onSettled(true)); + probe.connect(port, host, () => + probe.write("GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"), + ); + return Effect.void; + }).pipe(Effect.timeout("5 seconds")), + ), + ); + +it.live( + "lets a response already in flight when stopAccepting runs finish and deliver its full body", + () => + Effect.scoped( + Effect.gen(function* () { + const backend = createServer((_incoming, outgoing) => { + outgoing.writeHead(200, { "content-type": "text/plain" }); + outgoing.write("first-chunk"); + // Held open deliberately: the test itself finishes the response, once stopAccepting + // has already run, so the response's generation crosses the freeze. + }); + const backendAddress = yield* listen(backend); + const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); + yield* proxy.setRoutes([ + { id: "api", prefix: "/", target: Effect.succeed(backendAddress) }, + ]); + + const heldResponseFiber = yield* Effect.callback( + (resume) => { + const onRequest = (_incoming: unknown, outgoing: ServerResponse) => + resume(Effect.succeed(outgoing)); + backend.once("request", onRequest); + return Effect.sync(() => backend.off("request", onRequest)); + }, + ).pipe(Effect.forkScoped); + const responseFiber = yield* request(proxy.port, "/", new Uint8Array()).pipe( + Effect.provide(NodeHttpClient.layerNodeHttp), + Effect.forkScoped, + ); + const heldResponse = yield* Fiber.join(heldResponseFiber); + + yield* proxy.stopAccepting; + expect(yield* connectionRefused(proxy.port, proxy.host)).toBe(true); + + heldResponse?.end("-second-chunk"); + const response = yield* Fiber.join(responseFiber); + expect(response.status).toBe(200); + expect(new TextDecoder().decode(response.body)).toBe("first-chunk-second-chunk"); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live( + "refuses a new connection after stopAccepting while an established connection keeps completing requests until it closes", + () => + Effect.scoped( + Effect.gen(function* () { + let requests = 0; + const backend = createServer((_request, response) => { + requests += 1; + response.end(requests === 1 ? "first-response" : "second-response"); + }); + const backendAddress = yield* listen(backend); + const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); + yield* proxy.setRoutes([ + { id: "api", prefix: "/", target: Effect.succeed(backendAddress) }, + ]); + + const client = yield* connectRaw(proxy.port, proxy.host); + client.write("GET / HTTP/1.1\r\nHost: localhost\r\n\r\n"); + const first = yield* readFullResponse(client); + expect(first.status).toBe(200); + expect(first.body).toBe("first-response"); + expect(yield* SubscriptionRef.get(proxy.outstandingConnections)).toBe(1); + + yield* proxy.stopAccepting; + + expect(yield* connectionRefused(proxy.port, proxy.host)).toBe(true); + + client.write("GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"); + const second = yield* readFullResponse(client); + expect(second.status).toBe(200); + expect(second.body).toBe("second-response"); + + const reachedZero = yield* Stream.runHead( + Stream.filter( + SubscriptionRef.changes(proxy.outstandingConnections), + (count) => count === 0, + ), + ).pipe(Effect.forkScoped); + yield* Effect.sync(() => client.end()); + yield* Fiber.join(reachedZero); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("destroys every established connection immediately when cutAll runs", () => + Effect.scoped( + Effect.gen(function* () { + const backend = createServer((_request, response) => response.end("ok")); + const backendAddress = yield* listen(backend); + const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); + yield* proxy.setRoutes([{ id: "api", prefix: "/", target: Effect.succeed(backendAddress) }]); + + const client = yield* connectRaw(proxy.port, proxy.host); + client.write("GET / HTTP/1.1\r\nHost: localhost\r\n\r\n"); + const response = yield* readFullResponse(client); + expect(response.status).toBe(200); + expect(response.body).toBe("ok"); + expect(yield* SubscriptionRef.get(proxy.outstandingConnections)).toBe(1); + + const closed = yield* Effect.callback((resume) => { + const onClose = () => resume(Effect.void); + client.once("close", onClose); + if (client.destroyed) onClose(); + return Effect.sync(() => client.off("close", onClose)); + }).pipe(Effect.timeout("5 seconds"), Effect.forkScoped); + yield* proxy.cutAll; + yield* Fiber.join(closed); + expect(yield* SubscriptionRef.get(proxy.outstandingConnections)).toBe(0); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index d2ae57d119..c896997818 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -1,4 +1,4 @@ -import { Data, Effect, FiberSet, Ref, Scope } from "effect"; +import { Data, Effect, FiberSet, Ref, Scope, SubscriptionRef } from "effect"; import { PortError } from "./Ports.ts"; import type { BackendAddress, ProxyError } from "./Proxy.ts"; import { @@ -45,6 +45,12 @@ export interface HttpProxy { readonly host: string; readonly port: number; readonly setRoutes: (routes: ReadonlyArray) => Effect.Effect; + /** Stops accepting new connections on the listener; established connections keep flowing. */ + readonly stopAccepting: Effect.Effect; + /** The count of established client connections, observable until it reaches 0. */ + readonly outstandingConnections: SubscriptionRef.SubscriptionRef; + /** Destroys every established connection immediately. */ + readonly cutAll: Effect.Effect; } const hopByHop = new Set([ @@ -414,7 +420,12 @@ export const makeHttpProxy = (options: { (value) => Effect.sync(() => value.destroy()), ); const runRequest = yield* FiberSet.makeRuntime(); + const services = yield* Effect.context(); const sockets = new Set(); + const outstandingConnections = yield* SubscriptionRef.make(0); + // `server.close()` also tears down idle keep-alive sockets immediately, which would end + // established connections instead of letting them keep flowing; track acceptance separately. + let accepting = true; const server = createServer((request, response) => { runRequest( Effect.scoped( @@ -454,8 +465,20 @@ export const makeHttpProxy = (options: { ); }); server.on("connection", (socket) => { + if (!accepting) { + socket.destroy(); + return; + } sockets.add(socket); - socket.once("close", () => sockets.delete(socket)); + Effect.runSyncWith(services)( + SubscriptionRef.update(outstandingConnections, (count) => count + 1), + ); + socket.once("close", () => { + sockets.delete(socket); + Effect.runSyncWith(services)( + SubscriptionRef.update(outstandingConnections, (count) => count - 1), + ); + }); }); server.on("upgrade", (request, socket, head) => { socket.on("error", () => socket.destroy()); @@ -505,5 +528,12 @@ export const makeHttpProxy = (options: { routes, [...next].sort((a, b) => b.prefix.length - a.prefix.length), ), + stopAccepting: Effect.sync(() => { + accepting = false; + }), + outstandingConnections, + cutAll: Effect.sync(() => { + for (const socket of sockets) socket.destroy(); + }), }; }); diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index b4017a60ed..725c5375cc 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -200,9 +200,6 @@ const makeNetwork = (options: { yield* Effect.forkIn( serveTcp(listener, endpoint.backend, `${id}:${name}`).pipe( Effect.provideService(Scope.Scope, endpointScope), - Effect.catch((cause) => - Effect.logWarning("Public listener failed", cause), - ), ), endpointScope, { startImmediately: true }, diff --git a/packages/stack/src/Owner.integration.test.ts b/packages/stack/src/Owner.integration.test.ts index 05c8efe356..ddb48eed3b 100644 --- a/packages/stack/src/Owner.integration.test.ts +++ b/packages/stack/src/Owner.integration.test.ts @@ -1,17 +1,22 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { PgClient } from "@effect/sql-pg"; import { expect, it } from "@effect/vitest"; -import { Context, Effect, FileSystem, Layer, Redacted, Ref, Schema } from "effect"; +import { Context, Effect, FileSystem, Layer, Redacted, Ref, Schema, Stream } from "effect"; import { HttpClient } from "effect/unstable/http"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { randomUUID } from "node:crypto"; import { tmpdir } from "node:os"; import { RpcTest } from "effect/unstable/rpc"; import * as Owner from "./Owner.ts"; import { OwnerRpc } from "./Rpc.ts"; +import * as PortReservations from "./namespace/PortReservations.ts"; import * as StackNamespace from "./StackNamespace.ts"; import type { SavedStack } from "./StackNamespace.ts"; import { DEFAULT_LOCAL_JWT_SECRET } from "./Defaults.ts"; import { ServiceCreation, type ServiceCreationInput } from "./services/Catalog.ts"; import { ownerFor } from "../tests/owner-rpc.ts"; +import { dockerEngineTarget } from "../tests/engine-target.ts"; +import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; const stateFor = (root: string) => Effect.gen(function* () { @@ -31,6 +36,61 @@ const initial = (id: string): SavedStack => ({ ports: [], }); +const initialDocker = (id: string): SavedStack => ({ ...initial(id), runtime: "docker" }); + +/** Container ids docker still reports for `stackId`, by the label every instance container carries. */ +/** + * Container ids docker still reports for `instanceId`, by the labels an instance's own containers + * carry. Excludes a stack-wide shared volume helper (labeled only with the stack, never the + * instance), which outlives any one instance's destroy and is torn down on its own lifecycle. + */ +const containersForInstance = (stackId: string, instanceId: string) => + Effect.scoped( + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + ChildProcess.make( + "docker", + [ + "ps", + "-a", + "--filter", + `label=com.supabase.stack=${stackId}`, + "--filter", + `label=com.supabase.instance=${instanceId}`, + "--format", + "{{.ID}} {{.Names}} {{.Image}} {{.Status}}", + ], + { stdout: "pipe", stderr: "pipe" }, + ), + ); + const output = yield* Stream.mkString(Stream.decodeText(child.stdout)); + return output + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); + }), + ).pipe(Effect.provide(NodeServices.layer)); + +/** Runs a Docker CLI command and returns its combined stdout, for assertions against the engine. */ +const runDocker = (args: ReadonlyArray) => + Effect.scoped( + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + ChildProcess.make("docker", args, { stdout: "pipe", stderr: "pipe" }), + ); + return yield* Stream.mkString(Stream.decodeText(child.stdout)); + }), + ).pipe(Effect.provide(NodeServices.layer)); + +/** The fields read directly off a database instance's own storage marker file. */ +const InstanceMarker = Schema.Struct({ + backend: Schema.Literals(["docker", "host"]), + volume: Schema.optionalKey(Schema.String), + namespace: Schema.String, +}); + const query = (url: string, statement: string) => Effect.scoped( Effect.gen(function* () { @@ -826,3 +886,161 @@ it.live("rejects a missing required input before starting or stopping the servic }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); + +it.live( + "destroys a database's container, storage namespace and port reservation after its registration file is deleted", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const stackId = `owner-no-registration-${randomUUID().slice(0, 8)}`; + const dataRoot = yield* makeDockerDatabaseRoot("stack-owner-no-registration-", stackId); + const tempRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-owner-no-registration-state-", + }); + const stack = initialDocker(stackId); + const state = yield* stateFor(`${tempRoot}/state`); + yield* state.save(stack); + const owner = yield* ownerFor({ + saved: stack, + state, + root: dataRoot, + cacheRoot, + engineTarget: dockerEngineTarget, + }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + + const created = yield* owner.rpc.createService({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("owner-no-registration-password"), + jwtSecret: Redacted.make("owner-no-registration-jwt-secret-at-least-32-characters"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* owner.rpc.startService({ id: created.id }); + yield* owner.rpc.readyService({ id: created.id }); + + const realStateRoot = yield* fs.realPath(`${tempRoot}/state`); + const portReservations = yield* PortReservations.Service.pipe( + Effect.provide(PortReservations.layer), + ); + expect( + yield* portReservations.find(realStateRoot, stackId, `${created.id}:sql`), + ).toBeDefined(); + expect(yield* containersForInstance(stackId, created.id)).not.toHaveLength(0); + + // The registration disappears from under the live owner (deleted externally, or by a + // concurrent process); its resource cleanup must still run to completion and release + // everything it owns, with the removal simply having nothing left to publish. + yield* fs.remove(`${tempRoot}/state/${stackId}/state.json`); + + yield* owner.rpc.destroyService({ id: created.id }); + + expect(yield* containersForInstance(stackId, created.id)).toHaveLength(0); + expect( + yield* portReservations.find(realStateRoot, stackId, `${created.id}:sql`), + ).toBeUndefined(); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), + 120_000, +); + +it.live( + "destroys a database's container and storage namespace, and releases its port reservation, after the whole data root is deleted", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const stackId = `owner-no-data-root-${randomUUID().slice(0, 8)}`; + const dataRoot = yield* makeDockerDatabaseRoot("stack-owner-no-data-root-", stackId); + const tempRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-owner-no-data-root-state-", + }); + const stack = initialDocker(stackId); + const state = yield* stateFor(`${tempRoot}/state`); + yield* state.save(stack); + const owner = yield* ownerFor({ + saved: stack, + state, + root: dataRoot, + cacheRoot, + engineTarget: dockerEngineTarget, + }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + + const created = yield* owner.rpc.createService({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("owner-no-data-root-password"), + jwtSecret: Redacted.make("owner-no-data-root-jwt-secret-at-least-32-characters"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* owner.rpc.startService({ id: created.id }); + yield* owner.rpc.readyService({ id: created.id }); + + const realStateRoot = yield* fs.realPath(`${tempRoot}/state`); + const portReservations = yield* PortReservations.Service.pipe( + Effect.provide(PortReservations.layer), + ); + expect( + yield* portReservations.find(realStateRoot, stackId, `${created.id}:sql`), + ).toBeDefined(); + expect(yield* containersForInstance(stackId, created.id)).not.toHaveLength(0); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(InstanceMarker))( + yield* fs.readFileString(`${dataRoot}/${created.id}/.supabase-database-storage.json`), + ); + if (marker.backend !== "docker" || marker.volume === undefined) + return yield* Effect.die("Docker test selected host fallback"); + const volume = marker.volume; + + // The database sleeps; its storage object (inside the still-running owner) keeps the + // identity it already resolved in memory, then the whole owned data root disappears (the + // volume host, a disk failure, manual cleanup). + yield* owner.rpc.stopService({ id: created.id }); + expect(yield* containersForInstance(stackId, created.id)).toHaveLength(0); + yield* fs.remove(dataRoot, { recursive: true, force: true }); + + yield* owner.rpc.destroyService({ id: created.id }); + + expect(yield* containersForInstance(stackId, created.id)).toHaveLength(0); + expect( + yield* portReservations.find(realStateRoot, stackId, `${created.id}:sql`), + ).toBeUndefined(); + const remainingNamespace = yield* runDocker([ + "run", + "--rm", + "--mount", + `type=volume,src=${volume},dst=/store`, + "busybox:1.36", + "/bin/sh", + "-c", + `[ -e /store/${marker.namespace} ] && echo present || echo absent`, + ]); + expect(remainingNamespace.trim()).toBe("absent"); + yield* runDocker(["volume", "rm", volume]); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), + 120_000, +); diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index ee67ca736e..b3bf0d688e 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -193,6 +193,14 @@ interface Entry extends Orchestrator.RegisteredInstance { /** The saved creation, which composition wiring and launches update. */ readonly creation: Ref.Ref; readonly namespace: NetworkNamespace; + /** + * Removes this instance's containers, storage namespace and network namespace, independent of + * the saved registration: used by destroy (which then publishes the registration removal) and + * reusable as-is by a future abandonment path, which releases without touching the registration. + */ + readonly cleanupResources: ( + context: ServiceInstanceContext, + ) => Effect.Effect; } const isRecord = (value: unknown): value is Readonly> => @@ -317,6 +325,19 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { options.state.withLock( readSaved.pipe(Effect.flatMap((current) => options.state.save(update(current)))), ); + // Idempotent on an already-missing registration (a deleted or already-destroyed state.json), + // unlike `updateState`: destroy must still be able to publish an instance's removal, or confirm + // there is nothing left to publish, even when the registration disappeared from under it. + const removeInstanceRegistration = (id: string) => + options.state.withLock( + options.state + .read(stackId) + .pipe( + Effect.flatMap((current) => + current === undefined ? Effect.void : options.state.save(withoutInstance(current, id)), + ), + ), + ); const requireStopped = (configuration: CompositionConfig) => Effect.forEach( @@ -401,6 +422,20 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { const initial = recipe.creation; const creation = yield* Ref.make(initial); const namespaceRef = yield* Ref.make(undefined); + // Independent of the saved registration and of on-disk discovery: removes containers and the + // owned storage namespace, then releases the network namespace. Destroy below runs this to + // completion before publishing the registration removal, so a confirmed removal is never + // recorded ahead of the resources it describes; a future abandonment path can call this same + // operation and release the stack's ports without ever reading or saving the registration. + const cleanupResources = (context: ServiceInstanceContext) => + recipe.definition.removeData(context).pipe( + Effect.andThen( + Ref.get(namespaceRef).pipe( + Effect.flatMap((namespace) => namespace?.release ?? Effect.void), + Effect.mapError(serviceError("release")), + ), + ), + ); const core = yield* makeService( { ...recipe.definition, @@ -410,17 +445,9 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { Effect.andThen(recipe.definition.launch(context)), ), removeData: (context) => - recipe.definition.removeData(context).pipe( - Effect.andThen( - Ref.get(namespaceRef).pipe( - Effect.flatMap((namespace) => namespace?.release ?? Effect.void), - Effect.mapError(serviceError("release")), - ), - ), + cleanupResources(context).pipe( Effect.andThen( - updateState((current) => withoutInstance(current, id)).pipe( - Effect.mapError(serviceError("state")), - ), + removeInstanceRegistration(id).pipe(Effect.mapError(serviceError("state"))), ), ), }, @@ -475,6 +502,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { recipe, creation, namespace, + cleanupResources, startAt: (revision, inputs, wake, guard) => Ref.get(creation).pipe( Effect.flatMap((current) => mergeInputs(current, inputs)), diff --git a/packages/stack/src/Proxy.integration.test.ts b/packages/stack/src/Proxy.integration.test.ts index e08b81eaa2..bcc3d4e892 100644 --- a/packages/stack/src/Proxy.integration.test.ts +++ b/packages/stack/src/Proxy.integration.test.ts @@ -1,6 +1,16 @@ import { NodeHttpClient, NodeHttpServer } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Deferred, Effect, Fiber, Predicate } from "effect"; +import { + Deferred, + Effect, + Exit, + Fiber, + Predicate, + Ref, + Scope, + Stream, + SubscriptionRef, +} from "effect"; import { HttpClient, HttpClientRequest, @@ -149,3 +159,241 @@ it.live("does not log an error when a backend copy resets after a successful con }), ).pipe(Effect.provide(captureErrors(logs))); }); + +/** Starts a raw TCP echo backend and returns its address, retained until the scope closes. */ +const echoBackend = () => + Effect.acquireRelease( + Effect.callback< + { readonly server: Server; readonly host: string; readonly port: number }, + never + >((resume) => { + const server = createServer((socket) => socket.pipe(socket)); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + resume( + Effect.succeed({ + server, + host: "127.0.0.1", + port: typeof address === "object" && address !== null ? address.port : 0, + }), + ); + }); + return Effect.void; + }), + ({ server }) => + Effect.callback((resume) => { + server.close(() => resume(Effect.void)); + return Effect.void; + }), + ); + +const connectAndWrite = (port: number, message: string) => + Effect.acquireRelease( + Effect.callback((resume) => { + const socket = new Socket(); + socket.once("connect", () => resume(Effect.succeed(socket))); + socket.connect(port, "127.0.0.1"); + return Effect.void; + }), + (socket) => Effect.sync(() => socket.destroy()), + ).pipe(Effect.tap((socket) => Effect.sync(() => socket.write(message)))); + +const readOnceTcp = (socket: Socket) => + Effect.callback((resume) => { + const onData = (chunk: Buffer) => resume(Effect.succeed(chunk.toString())); + socket.once("data", onData); + return Effect.sync(() => socket.off("data", onData)); + }).pipe(Effect.timeout("5 seconds")); + +it.live( + "releases the scoped target and keeps serving the next connection after a client resets while readiness is pending", + () => + Effect.scoped( + Effect.gen(function* () { + const listener = yield* bindTcp("127.0.0.1", 0); + if (!Predicate.isTagged(listener.address, "TcpAddress")) + return yield* Effect.die("Expected TCP listener"); + const port = listener.address.port; + const backend = yield* echoBackend(); + const acquired = yield* Deferred.make(); + const released = yield* Deferred.make(); + const gate = yield* Deferred.make(); + const connections = yield* Ref.make(0); + // Only the first connection's target acquisition is held behind the gate; a later one + // (the "listener still serves" check below) resolves immediately against the real backend. + const target = Ref.updateAndGet(connections, (count) => count + 1).pipe( + Effect.flatMap((count) => + count === 1 + ? Effect.acquireRelease(Deferred.succeed(acquired, undefined), () => + Deferred.succeed(released, undefined), + ).pipe( + Effect.andThen(Deferred.await(gate)), + Effect.as({ host: backend.host, port: backend.port }), + ) + : Effect.succeed({ host: backend.host, port: backend.port }), + ), + ); + yield* serveTcp(listener, target, "cold-wake").pipe(Effect.forkScoped); + + const client = yield* Effect.acquireRelease( + Effect.callback((resume) => { + const socket = new Socket(); + socket.once("connect", () => resume(Effect.succeed(socket))); + socket.connect(port, "127.0.0.1"); + return Effect.void; + }), + (socket) => Effect.sync(() => socket.destroy()), + ); + yield* Deferred.await(acquired); + // Readiness is still pending behind `gate`; an unhandled reset here would otherwise crash + // the owning process before the fix. + yield* Effect.sync(() => client.resetAndDestroy()); + yield* Deferred.await(released); + + // The listener (and the process running it) survived the reset and still serves a fresh + // connection normally. + const probe = yield* connectAndWrite(port, "ping"); + expect(yield* readOnceTcp(probe)).toBe("ping"); + }), + ), +); + +it.live( + "refuses a new connection after stopAccepting while an established connection keeps relaying data through the scoped target until it closes", + () => + Effect.scoped( + Effect.gen(function* () { + const backend = yield* echoBackend(); + const listener = yield* bindTcp("127.0.0.1", 0); + if (!Predicate.isTagged(listener.address, "TcpAddress")) + return yield* Effect.die("Expected TCP listener"); + const target = Effect.succeed({ host: backend.host, port: backend.port }); + yield* serveTcp(listener, target, "echo").pipe(Effect.forkScoped); + const port = listener.address.port; + + const client = yield* connectAndWrite(port, "ping"); + expect(yield* readOnceTcp(client)).toBe("ping"); + expect(yield* SubscriptionRef.get(listener.outstandingConnections)).toBe(1); + + yield* listener.stopAccepting; + + const refused = yield* Effect.callback((resume) => { + const probe = new Socket(); + let responded = false; + let settled = false; + // A refused connection surfaces as a reset, a generic error, or a plain close; all three + // are the allowed refusal outcomes here, so long as no response ever arrived. + const settle = () => { + if (settled) return; + settled = true; + resume(Effect.succeed(!responded)); + }; + probe.once("data", () => { + responded = true; + }); + probe.once("error", settle); + probe.once("close", settle); + probe.connect(port, "127.0.0.1", () => probe.write("probe")); + return Effect.sync(() => probe.destroy()); + }).pipe(Effect.timeout("5 seconds")); + expect(refused).toBe(true); + + client.write("pong"); + expect(yield* readOnceTcp(client)).toBe("pong"); + + const reachedZero = yield* Stream.runHead( + Stream.filter( + SubscriptionRef.changes(listener.outstandingConnections), + (count) => count === 0, + ), + ).pipe(Effect.forkScoped); + yield* Effect.sync(() => client.end()); + yield* Fiber.join(reachedZero); + }), + ), +); + +it.live( + "destroys every established connection immediately when cutAll runs on the production listener", + () => + Effect.scoped( + Effect.gen(function* () { + const backend = yield* echoBackend(); + const listener = yield* bindTcp("127.0.0.1", 0); + if (!Predicate.isTagged(listener.address, "TcpAddress")) + return yield* Effect.die("Expected TCP listener"); + const target = Effect.succeed({ host: backend.host, port: backend.port }); + yield* serveTcp(listener, target, "echo").pipe(Effect.forkScoped); + const port = listener.address.port; + + const client = yield* connectAndWrite(port, "ping"); + expect(yield* readOnceTcp(client)).toBe("ping"); + expect(yield* SubscriptionRef.get(listener.outstandingConnections)).toBe(1); + + const closed = yield* Effect.callback((resume) => { + const onClose = () => resume(Effect.void); + client.once("close", onClose); + if (client.destroyed) onClose(); + return Effect.sync(() => client.off("close", onClose)); + }).pipe(Effect.timeout("5 seconds"), Effect.forkScoped); + yield* listener.cutAll; + yield* Fiber.join(closed); + expect(yield* SubscriptionRef.get(listener.outstandingConnections)).toBe(0); + }), + ), +); + +it.live( + "counts, cuts, and tears down a connection that arrives before run installs its handler", + () => + Effect.gen(function* () { + const listenerScope = yield* Scope.make(); + const listener = yield* bindTcp("127.0.0.1", 0).pipe(Scope.provide(listenerScope)); + if (!Predicate.isTagged(listener.address, "TcpAddress")) + return yield* Effect.die("Expected TCP listener"); + const port = listener.address.port; + + const connect = () => + Effect.callback((resume) => { + const socket = new Socket(); + socket.once("connect", () => resume(Effect.succeed(socket))); + socket.connect(port, "127.0.0.1"); + return Effect.void; + }).pipe(Effect.timeout("5 seconds")); + const awaitClose = (socket: Socket) => + Effect.callback((resume) => { + const onClose = () => resume(Effect.void); + socket.once("close", onClose); + if (socket.destroyed) onClose(); + return Effect.sync(() => socket.off("close", onClose)); + }).pipe(Effect.timeout("5 seconds")); + // The server's own acceptance (tracked by `outstandingConnections`) is a separate event from + // the client's own "connect"; subscribed before connecting so neither ordering is assumed. + const awaitOutstanding = (expected: number) => + Stream.runHead( + Stream.filter( + SubscriptionRef.changes(listener.outstandingConnections), + (count) => count === expected, + ), + ).pipe(Effect.timeout("5 seconds")); + + // `run` is never called on this listener: every connection below is, and stays, queued. + const firstAccepted = yield* awaitOutstanding(1).pipe(Effect.forkScoped); + const first = yield* connect(); + yield* Fiber.join(firstAccepted); + + const firstCut = yield* awaitOutstanding(0).pipe(Effect.forkScoped); + yield* listener.cutAll; + yield* awaitClose(first); + yield* Fiber.join(firstCut); + + // Closing the listener's own scope destroys a still-queued connection and completes rather + // than hanging on `server.close()` waiting for a socket nothing ever destroyed. + const secondAccepted = yield* awaitOutstanding(1).pipe(Effect.forkScoped); + const second = yield* connect(); + yield* Fiber.join(secondAccepted); + const secondClosed = yield* awaitClose(second).pipe(Effect.forkScoped); + yield* Scope.close(listenerScope, Exit.void).pipe(Effect.timeout("5 seconds")); + yield* Fiber.join(secondClosed); + }).pipe(Effect.scoped), +); diff --git a/packages/stack/src/Proxy.ts b/packages/stack/src/Proxy.ts index 6003d55cd9..9faaa36513 100644 --- a/packages/stack/src/Proxy.ts +++ b/packages/stack/src/Proxy.ts @@ -1,6 +1,5 @@ -import { NodeSink, NodeSocket, NodeSocketServer, NodeStream } from "@effect/platform-node"; -import { Data, Effect, Option, Stream } from "effect"; -import type { Scope } from "effect"; +import { NodeSink, NodeStream } from "@effect/platform-node"; +import { Data, Effect, Exit, FiberSet, Scope, Stream, SubscriptionRef } from "effect"; import type { SocketServer } from "effect/unstable/socket"; import * as Net from "node:net"; import { PortError } from "./Ports.ts"; @@ -17,20 +16,143 @@ export class ProxyError extends Data.TaggedError("ProxyError")<{ const proxyError = (cause: unknown) => new ProxyError({ message: cause instanceof Error ? cause.message : String(cause), cause }); -/** Binds a dedicated public listener and retains the socket until its scope closes. */ -export const bindTcp = (host: string, port: number) => - NodeSocketServer.make({ host, port, allowHalfOpen: true }).pipe( - Effect.mapError( - (cause) => - // `SocketServerError.reason.cause` is the OS error (for example EADDRINUSE); unwrap it - // here so `Ports.ts`'s conflict detection sees the same shape it gets from `HttpProxy`. - new PortError({ - key: "tcp", - message: "Cannot bind TCP listener", - cause: cause.reason.cause, +export interface TcpListener { + readonly address: SocketServer.Address; + /** Installs the per-connection handler; keeps the listener alive until interrupted. */ + readonly run: ( + handler: (socket: Net.Socket) => Effect.Effect<_, E, R>, + ) => Effect.Effect; + /** Stops accepting new connections on the listener; established connections keep flowing. */ + readonly stopAccepting: Effect.Effect; + /** The count of established client connections, observable until it reaches 0. */ + readonly outstandingConnections: SubscriptionRef.SubscriptionRef; + /** Destroys every established connection immediately. */ + readonly cutAll: Effect.Effect; +} + +/** + * Binds a dedicated public listener and retains the socket until its scope closes. Built directly + * on `node:net`, not `NodeSocketServer`, because `NodeSocketServer`'s `run` only exposes a combined + * accept-and-destroy teardown: this separates closing accept (`stopAccepting`) from cutting + * established connections (`cutAll`), with the count of outstanding ones observable in between. + */ +export const bindTcp = ( + host: string, + port: number, +): Effect.Effect => + Effect.gen(function* () { + const services = yield* Effect.context(); + // Every accepted socket, queued or dispatched, from acceptance until close: the single source + // for the outstanding count, `cutAll`, and teardown, so none of them is ever only half-tracked. + const sockets = new Set(); + const outstandingConnections = yield* SubscriptionRef.make(0); + let accepting = true; + // A connection that arrives before `run` installs its handler is queued, the same as + // `NodeSocketServer`'s own listener, so none are dropped in that gap. + const pending = new Set(); + let onConnection: (conn: Net.Socket) => void = (conn) => { + pending.add(conn); + }; + const server = Net.createServer({ allowHalfOpen: true }, (conn) => { + // Attached synchronously, before any dispatch, so a reset while a handler is still cold + // (queued, or waiting on scoped target acquisition) never surfaces as an unhandled error. + conn.on("error", () => conn.destroy()); + if (!accepting) { + conn.destroy(); + return; + } + sockets.add(conn); + Effect.runSyncWith(services)( + SubscriptionRef.update(outstandingConnections, (count) => count + 1), + ); + conn.once("close", () => { + sockets.delete(conn); + pending.delete(conn); + Effect.runSyncWith(services)( + SubscriptionRef.update(outstandingConnections, (count) => count - 1), + ); + }); + onConnection(conn); + }); + yield* Effect.acquireRelease( + Effect.callback((resume) => { + const onError = (cause: Error) => + // Unwrapped so `Ports.ts`'s conflict detection sees the same shape it gets from + // `HttpProxy` (for example EADDRINUSE). + resume( + Effect.fail(new PortError({ key: "tcp", message: "Cannot bind TCP listener", cause })), + ); + server.once("error", onError); + server.listen(port, host, () => { + // Accept errors after binding (for example fd exhaustion) are logged for the listener's + // whole life rather than left unhandled to crash the owner. + server.off("error", onError); + server.on("error", (cause) => + Effect.runSyncWith(services)(Effect.logError("TCP listener error", cause)), + ); + resume(Effect.void); + }); + return Effect.sync(() => server.off("error", onError)); + }), + () => + Effect.callback((resume) => { + pending.clear(); + for (const socket of sockets) socket.destroy(); + server.close(() => resume(Effect.void)); + return Effect.void; }), - ), - ); + ); + const bound = server.address(); + if (bound === null) + return yield* new PortError({ key: "tcp", message: "TCP listener has no address" }); + const address: SocketServer.Address = + typeof bound === "string" + ? { _tag: "UnixAddress", path: bound } + : { _tag: "TcpAddress", hostname: bound.address, port: bound.port }; + + const run = ( + handler: (socket: Net.Socket) => Effect.Effect<_, E, R>, + ): Effect.Effect => + Effect.gen(function* () { + const connectionScope = yield* Scope.make(); + const runFiber = yield* FiberSet.makeRuntime().pipe( + Effect.provideService(Scope.Scope, connectionScope), + ); + const previous = onConnection; + onConnection = (conn) => { + runFiber( + handler(conn).pipe( + Effect.catchCause((cause) => + Effect.logError("Unhandled TCP connection failure", cause), + ), + ), + ); + }; + for (const conn of pending) onConnection(conn); + pending.clear(); + return yield* Effect.onExit(Effect.never, () => + Scope.close(connectionScope, Exit.void).pipe( + Effect.andThen( + Effect.sync(() => { + onConnection = previous; + }), + ), + ), + ); + }); + + return { + address, + run, + stopAccepting: Effect.sync(() => { + accepting = false; + }), + outstandingConnections, + cutAll: Effect.sync(() => { + for (const socket of sockets) socket.destroy(); + }), + }; + }); const connect = Effect.fn("Proxy.connect")((address: BackendAddress) => Effect.gen(function* () { @@ -64,22 +186,13 @@ const copy = (source: Net.Socket, destination: Net.Socket) => /** The scoped target acquisition owns request activity and performs orchestrated wake/readiness. */ export const serveTcp = Effect.fn("Proxy.serveTcp")( ( - listener: SocketServer.SocketServer["Service"], + listener: TcpListener, target: Effect.Effect, label: string, ) => - listener.run(() => + listener.run((incoming) => Effect.scoped( Effect.gen(function* () { - // rc.112 supplies this service to handlers but does not remove it from run's requirements. - const incoming = yield* Effect.serviceOption(NodeSocket.NetSocket).pipe( - Effect.flatMap( - Option.match({ - onNone: () => Effect.fail(proxyError("TCP listener supplied no connection")), - onSome: Effect.succeed, - }), - ), - ); yield* Effect.addFinalizer(() => Effect.sync(() => { incoming.destroy(); diff --git a/packages/stack/src/Service.integration.test.ts b/packages/stack/src/Service.integration.test.ts index 2769658da9..5947e64b4f 100644 --- a/packages/stack/src/Service.integration.test.ts +++ b/packages/stack/src/Service.integration.test.ts @@ -446,6 +446,53 @@ describe("service kernel", () => { ), ); + it.live( + "stops and cleans up the running workload when a restart is destroyed mid-preparation and completes late", + () => + Effect.scoped( + Effect.gen(function* () { + const plans = yield* Queue.unbounded(); + const preparations = yield* Queue.unbounded(); + const initialPreparation = yield* makePreparationPlan; + const restartPreparation = yield* makePreparationPlan; + yield* open(initialPreparation.gate); + yield* Queue.offer(preparations, initialPreparation); + yield* Queue.offer(preparations, restartPreparation); + const service = yield* makeService(makeDefinition(plans, preparations), { + id: "database-late-restart", + config: { version: 17 }, + }); + const original = yield* makeRuntimePlan; + const superseded = yield* makeRuntimePlan; + yield* Queue.offer(plans, original); + yield* Queue.offer(plans, superseded); + yield* open(original.launchGate); + yield* service.start; + expect((yield* service.get).lifecycle).toBe("running"); + + const restarting = yield* service.restart({ version: 18 }).pipe(Effect.forkScoped); + yield* Deferred.await(restartPreparation.started); + + yield* open(original.stopGate); + yield* open(original.removeGate); + yield* service.destroy; + expect((yield* service.get).registered).toBe(false); + expect((yield* Ref.get(original.state)).removed).toBe(true); + + // The superseded restart's own preparation only resolves after the destroy above, so its + // launch attempt (if it ran at all) would be the one completing late. + yield* open(restartPreparation.gate); + const result = yield* Fiber.await(restarting); + expect(Exit.isFailure(result)).toBe(true); + if (Exit.isFailure(result)) + expect(result.cause.reasons.find(Cause.isFailReason)?.error).toBeInstanceOf( + ServiceDestroyed, + ); + expect(yield* Deferred.isDone(superseded.launchStarted)).toBe(false); + }), + ), + ); + it.live("records a launch failure as stopped and can launch successfully on retry", () => Effect.scoped( Effect.gen(function* () { @@ -665,6 +712,52 @@ describe("service kernel", () => { ), ); + it.live( + "keeps the handle's scope and its resources alive through a cold launch after the caller is canceled", + () => + Effect.scoped( + Effect.gen(function* () { + const resourceClosed = yield* Deferred.make(); + const launchStarted = yield* Deferred.make(); + const launchGate = yield* Deferred.make(); + const service = yield* makeService( + { + launch: (context) => + Effect.gen(function* () { + // A real resource owned by the launch's execution handle, not just the fake + // plan's own bookkeeping: closing the handle early would settle this finalizer. + yield* Scope.addFinalizer( + context.scope, + Deferred.succeed(resourceClosed, undefined).pipe(Effect.asVoid), + ); + yield* Deferred.succeed(launchStarted, undefined); + yield* Deferred.await(launchGate); + return { + health: Effect.void, + exit: Effect.never, + stop: Effect.void, + remove: Effect.void, + } satisfies RuntimeSession; + }), + removeData: () => Effect.void, + }, + { id: "cold-launch-resource", config: { version: 1 } }, + ); + const caller = yield* service.start.pipe(Effect.forkScoped); + yield* Deferred.await(launchStarted); + yield* Fiber.interrupt(caller); + // The caller gave up mid-launch; the owner fiber and the scope it owns must stay live. + expect(yield* Deferred.isDone(resourceClosed)).toBe(false); + yield* Deferred.succeed(launchGate, undefined); + yield* service.ready; + expect((yield* service.get).lifecycle).toBe("running"); + expect(yield* Deferred.isDone(resourceClosed)).toBe(false); + yield* service.stop; + expect(yield* Deferred.isDone(resourceClosed)).toBe(true); + }), + ), + ); + it.live("lets independent instances progress concurrently", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index fbf3656f19..58ddfb6ace 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -145,6 +145,14 @@ interface SessionRecord { readonly removed: Ref.Ref; } +/** + * Owns one launch attempt's resources from before preparation through confirmed cleanup. A stale + * or abandoned attempt still closes its scope; a successful one hands that duty to `stopNow`. + */ +interface ExecutionHandle { + readonly scope: Scope.Closeable; +} + const sessionStopped = () => new ServiceError({ operation: "health", message: "Session stopped" }); const contextFor = ( @@ -191,6 +199,10 @@ export const makeService = ( const revision = yield* Ref.make(0); const launchCounter = yield* Ref.make(0); const current = yield* Ref.make(undefined); + // Forked before preparation starts so an abandoned or stale attempt still has a scope to close. + const allocateHandle: Effect.Effect = Scope.fork(owner, "parallel").pipe( + Effect.map((scope): ExecutionHandle => ({ scope })), + ); // stopNow clears observation.launchId while keeping the error, so readiness must not read that field. const launchError = yield* Ref.make< { readonly launchId: number; readonly error: ServiceError } | undefined @@ -270,11 +282,14 @@ export const makeService = ( ), ); - // Mask only the permit handoff; admitted work belongs to the host scope. + // Mask only the permit handoff; admitted work belongs to the host scope. `onAdmitted` runs + // uninterruptibly right after the fork succeeds, so a caller can transfer resource-cleanup + // ownership to the owner fiber atomically: once it has run, the caller's own interruption of + // the join below can no longer race ahead of that transfer. const run = Effect.fn("Service.run")(function* < A, E extends ServiceError | ServiceDestroyed | ServiceNotStopped, - >(operation: Effect.Effect) { + >(operation: Effect.Effect, onAdmitted: Effect.Effect = Effect.void) { return yield* Effect.uninterruptibleMask((restore) => Effect.gen(function* () { yield* restore(gate.take(1)); @@ -283,6 +298,7 @@ export const makeService = ( owner, { uninterruptible: false }, ); + yield* onAdmitted; return yield* restore(Fiber.join(fiber)); }), ); @@ -346,6 +362,8 @@ export const makeService = ( }); const launchNow = Effect.fn("Service.launchNow")(function* ( + handle: ExecutionHandle, + handedOff: Ref.Ref, guard: Effect.Effect, ) { yield* Effect.annotateCurrentSpan({ member_id: options.id }); @@ -380,7 +398,7 @@ export const makeService = ( ), ), ); - const runtimeScope = yield* Scope.fork(owner, "parallel"); + const runtimeScope = handle.scope; const launchExit = yield* Effect.exit( definition.launch(contextFor(options.id, yield* Ref.get(config), runtimeScope)).pipe( Effect.map((runtime) => ({ runtime, failure: undefined })), @@ -414,6 +432,8 @@ export const makeService = ( removed: yield* Ref.make(false), }; yield* Ref.set(current, record); + // From here, record.scope (== handle.scope) is stopNow's responsibility, not the caller's. + yield* Ref.set(handedOff, true); if (launchFailure !== undefined) yield* Ref.set(launchError, { launchId, error: launchFailure }); yield* update({ @@ -503,28 +523,58 @@ export const makeService = ( if (existing.lifecycle === "running") return; } const nextConfig = candidate ?? (yield* Ref.get(config)); - if (definition.prepare !== undefined) - yield* definition - .prepare(nextConfig) - .pipe(Effect.tapError((error) => recordPreparationFailure(expectedRevision, error))); - yield* run( - Effect.gen(function* () { - const observation = yield* SubscriptionRef.get(observations); - if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); - if (observation.lifecycle === "running" || observation.lifecycle === "starting") return; - if ((yield* Ref.get(revision)) !== expectedRevision || (wake && !observation.wakeEnabled)) - return yield* new ServiceError({ - operation: "admission", - message: "Service intent changed before admission", - }); - yield* setOperation("start"); - yield* launchNow( - guard.pipe( - Effect.andThen(Ref.set(config, nextConfig)), - Effect.andThen(update({ config: nextConfig })), + const handle = yield* allocateHandle; + const handedOff = yield* Ref.make(false); + // Set once `run` admits the operation onto the owner fiber; from that point the owner alone + // decides the handle's fate (via the inner `ensuring` below), atomically with the fork, so + // the caller's own interruption of the join can never race ahead of that decision. + const admitted = yield* Ref.make(false); + yield* Effect.gen(function* () { + if (definition.prepare !== undefined) + yield* definition + .prepare(nextConfig) + .pipe(Effect.tapError((error) => recordPreparationFailure(expectedRevision, error))); + yield* run( + Effect.gen(function* () { + const observation = yield* SubscriptionRef.get(observations); + if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); + if (observation.lifecycle === "running" || observation.lifecycle === "starting") return; + if ( + (yield* Ref.get(revision)) !== expectedRevision || + (wake && !observation.wakeEnabled) + ) + return yield* new ServiceError({ + operation: "admission", + message: "Service intent changed before admission", + }); + yield* setOperation("start"); + yield* launchNow( + handle, + handedOff, + guard.pipe( + Effect.andThen(Ref.set(config, nextConfig)), + Effect.andThen(update({ config: nextConfig })), + ), + ).pipe(Effect.ensuring(setOperation(undefined))); + }).pipe( + Effect.ensuring( + Ref.get(handedOff).pipe( + Effect.flatMap((isHandedOff) => + isHandedOff ? Effect.void : Scope.close(handle.scope, Exit.void), + ), + ), ), - ).pipe(Effect.ensuring(setOperation(undefined))); - }), + ), + Ref.set(admitted, true), + ); + }).pipe( + Effect.ensuring( + Ref.get(admitted).pipe( + Effect.flatMap((isAdmitted) => + isAdmitted ? Effect.void : Scope.close(handle.scope, Exit.void), + ), + ), + ), ); }); const start = Ref.get(revision).pipe(Effect.flatMap((revision) => startAt(revision))); @@ -596,25 +646,49 @@ export const makeService = ( yield* Effect.annotateCurrentSpan({ member_id: options.id }); const expectedRevision = requestedRevision ?? (yield* Ref.get(revision)); const nextConfig = candidate ?? (yield* Ref.get(config)); - if (definition.prepare !== undefined) yield* definition.prepare(nextConfig); - yield* run( - Effect.gen(function* () { - const observation = yield* SubscriptionRef.get(observations); - if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); - if ((yield* Ref.get(revision)) !== expectedRevision) - return yield* new ServiceError({ - operation: "admission", - message: "Service intent changed before admission", - }); - yield* coordinate("restart", invalidate()); - yield* setOperation("restart"); - yield* Effect.gen(function* () { - yield* stopNow(); - yield* Ref.set(config, nextConfig); - yield* update({ config: nextConfig }); - yield* launchNow(guard); - }).pipe(Effect.ensuring(setOperation(undefined))); - }), + const handle = yield* allocateHandle; + const handedOff = yield* Ref.make(false); + // See `startAt`: ownership transfers atomically to the owner fiber on admission, so the + // caller's own interruption of the join can never race ahead of that transfer. + const admitted = yield* Ref.make(false); + yield* Effect.gen(function* () { + if (definition.prepare !== undefined) yield* definition.prepare(nextConfig); + yield* run( + Effect.gen(function* () { + const observation = yield* SubscriptionRef.get(observations); + if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); + if ((yield* Ref.get(revision)) !== expectedRevision) + return yield* new ServiceError({ + operation: "admission", + message: "Service intent changed before admission", + }); + yield* coordinate("restart", invalidate()); + yield* setOperation("restart"); + yield* Effect.gen(function* () { + yield* stopNow(); + yield* Ref.set(config, nextConfig); + yield* update({ config: nextConfig }); + yield* launchNow(handle, handedOff, guard); + }).pipe(Effect.ensuring(setOperation(undefined))); + }).pipe( + Effect.ensuring( + Ref.get(handedOff).pipe( + Effect.flatMap((isHandedOff) => + isHandedOff ? Effect.void : Scope.close(handle.scope, Exit.void), + ), + ), + ), + ), + Ref.set(admitted, true), + ); + }).pipe( + Effect.ensuring( + Ref.get(admitted).pipe( + Effect.flatMap((isAdmitted) => + isAdmitted ? Effect.void : Scope.close(handle.scope, Exit.void), + ), + ), + ), ); }); diff --git a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts index 718c066403..57c629a2f9 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts @@ -1351,6 +1351,177 @@ describe("Docker database storage", { timeout: 120_000 }, () => { ).pipe(Effect.provide(NodeServices.layer)), ); + it.live( + "destroys the owned namespace in the shared volume after its on-disk marker is deleted, using the identity resolved in memory", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-no-marker-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "sleeping"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-no-marker-${yield* crypto.randomUUIDv4}`; + // One storage object spans the whole case, the way an Owner keeps one per instance for + // the stack's lifetime: its resolved identity stays in memory across the sleep below. + const storage = yield* makeDockerDatabaseStorage({ + claims: noContainerClaims, + runtime: "docker", + target: dockerTarget, + stackId, + instanceId: "sleeping", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + yield* storage.prepare("17"); + const markerPath = path.join(instanceRoot, ".supabase-database-storage.json"); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(markerPath), + ); + if (marker.backend !== "docker" || marker.volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + const volume = marker.volume; + const mount = yield* storage.mount("17"); + if (mount.type !== "volume" || mount.volumeSubpath === undefined) + return yield* new DockerTestError({ message: "Docker volume backend unavailable" }); + yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${volume},dst=/data,volume-subpath=${mount.volumeSubpath}`, + helperImage, + "/bin/sh", + "-c", + "printf 17 > /data/PG_VERSION", + ]); + yield* storage.markInitialized("17"); + + // The database sleeps (the registered instance stays stopped); its storage object keeps + // running, so this is the only change: the registration's on-disk marker is now gone. + yield* fs.remove(markerPath); + + yield* storage.destroyData("17"); + + const remaining = yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `[ -e ${quote(`/store/${marker.namespace}`)} ] && echo present || echo absent`, + ]); + expect(remaining.trim()).toBe("absent"); + yield* docker(["volume", "rm", volume]); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live( + "destroys the owned namespace after its whole state root is deleted, with no filesystem dependency for an identity already resolved in memory", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); + // `root` is deleted entirely below; nest it under the scope's own temp dir so the test + // framework's own cleanup of that outer, still-present directory does not fail. + const tempDir = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-no-root-" }); + const root = path.join(tempDir, "state-root"); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "sleeping"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + claims: noContainerClaims, + target: dockerTarget, + root, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-no-root-${yield* crypto.randomUUIDv4}`; + // One storage object spans the whole case, the way an Owner keeps one per instance for + // the stack's lifetime: its resolved identity stays in memory across the sleep below. + const storage = yield* makeDockerDatabaseStorage({ + claims: noContainerClaims, + runtime: "docker", + target: dockerTarget, + stackId, + instanceId: "sleeping", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + yield* storage.prepare("17"); + const markerPath = path.join(instanceRoot, ".supabase-database-storage.json"); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(markerPath), + ); + if (marker.backend !== "docker" || marker.volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + const volume = marker.volume; + const mount = yield* storage.mount("17"); + if (mount.type !== "volume" || mount.volumeSubpath === undefined) + return yield* new DockerTestError({ message: "Docker volume backend unavailable" }); + yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${volume},dst=/data,volume-subpath=${mount.volumeSubpath}`, + helperImage, + "/bin/sh", + "-c", + "printf 17 > /data/PG_VERSION", + ]); + yield* storage.markInitialized("17"); + + // The database sleeps; its storage object keeps running (the resolved identity is + // already in memory), then the whole state root disappears from under the live owner. + yield* fs.remove(root, { recursive: true, force: true }); + + yield* storage.destroyData("17"); + + const remaining = yield* docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `[ -e ${quote(`/store/${marker.namespace}`)} ] && echo present || echo absent`, + ]); + expect(remaining.trim()).toBe("absent"); + yield* docker(["volume", "rm", volume]); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + it.live("handles root-owned host data through helper operations", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index 7a693ff612..8c126db241 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -790,7 +790,13 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") .pipe(Effect.mapError((cause) => errorFor("marker", cause))); return present ? Option.some(yield* getMarker) : Option.none(); }); + // The in-memory resolution from `selected`, kept for the instance's lifetime across sleeps and + // generations once resolved, independent of the on-disk marker: a marker deleted after + // resolution must not hide a namespace this process still knows it owns. + const resolvedFromMemory = Ref.get(selectedCache); const getMarkerForRemoval = Effect.gen(function* () { + const cached = yield* resolvedFromMemory; + if (cached !== undefined) return Option.some(cached); const existing = yield* getMarkerIfPresent; if (Option.isSome(existing) && options.runtime === "docker") { yield* selected; @@ -798,6 +804,19 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") } return existing; }); + /** + * Destroy tolerates a missing volume by design (`validateDockerMarkerIdentity` plus its own + * volume-existence check below), so this skips `selected`'s stricter resolution and only adds + * the in-memory shortcut on top of the existing on-disk read. `validated` tells the caller + * whether `selected` already confirmed this identity in memory: a live owner's cleanup must + * not then re-derive it from the filesystem (`realPath(stateRoot)` in + * `validateDockerMarkerIdentity`), only a new owner discovering from disk needs that check. + */ + const getMarkerForDestroy = Effect.gen(function* () { + const cached = yield* resolvedFromMemory; + if (cached !== undefined) return { marker: Option.some(cached), validated: true as const }; + return { marker: yield* getMarkerIfPresent, validated: false as const }; + }); const readyMarkerPath = options.path.join( options.instanceRoot, ".supabase-database-ready.json", @@ -946,7 +965,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ); const destroyData = Effect.fn("DockerDatabaseStorage.destroyData")((version: string) => Effect.gen(function* () { - const markerOption = yield* getMarkerIfPresent; + const { marker: markerOption, validated } = yield* getMarkerForDestroy; if (Option.isNone(markerOption)) { yield* removeUnmarkedData(version, { checkpoints: true }); return yield* removeHelper(); @@ -962,7 +981,9 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* removeReadyMarker; yield* removeHelper(); } else { - yield* validateDockerMarkerIdentity(marker); + // Memory-sourced identities were already validated by `selected` when first resolved; + // only a marker discovered fresh from disk (a new owner recovering) needs this check. + if (!validated) yield* validateDockerMarkerIdentity(marker); if (marker.volume === undefined) return yield* errorFor("destroy", "Recorded Docker storage volume is missing"); const volumeExists = yield* engineCommand(["volume", "inspect", marker.volume]).pipe( From 9d260b35834e9056eb3bfa207d5b7f66d2ffd117 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sun, 4 Oct 2026 15:53:00 +0200 Subject: [PATCH 025/142] fix(stack): serialize artifact retirement with local pins and observe drain counts directly --- packages/stack/src/Proxy.integration.test.ts | 12 +++ packages/stack/src/namespace/Pin.ts | 21 +++- .../stack/src/preparation/ArtifactStore.ts | 34 ++++--- .../preparation/artifacts.integration.test.ts | 96 +++++++++++++++++++ 4 files changed, 149 insertions(+), 14 deletions(-) diff --git a/packages/stack/src/Proxy.integration.test.ts b/packages/stack/src/Proxy.integration.test.ts index bcc3d4e892..56f1248a31 100644 --- a/packages/stack/src/Proxy.integration.test.ts +++ b/packages/stack/src/Proxy.integration.test.ts @@ -336,8 +336,20 @@ it.live( if (client.destroyed) onClose(); return Effect.sync(() => client.off("close", onClose)); }).pipe(Effect.timeout("5 seconds"), Effect.forkScoped); + // Subscribed before `cutAll` runs: the client's own "close" event is a separate, cross- + // socket signal that reaches this process over the loopback connection, with no ordering + // guarantee relative to the listener's own bookkeeping (observed to lag it on Windows), so + // the listener's count must be awaited on its own observable, never inferred from the + // client's. + const reachedZero = yield* Stream.runHead( + Stream.filter( + SubscriptionRef.changes(listener.outstandingConnections), + (count) => count === 0, + ), + ).pipe(Effect.forkScoped); yield* listener.cutAll; yield* Fiber.join(closed); + yield* Fiber.join(reachedZero); expect(yield* SubscriptionRef.get(listener.outstandingConnections)).toBe(0); }), ), diff --git a/packages/stack/src/namespace/Pin.ts b/packages/stack/src/namespace/Pin.ts index 270bb88035..1461de46ee 100644 --- a/packages/stack/src/namespace/Pin.ts +++ b/packages/stack/src/namespace/Pin.ts @@ -1,4 +1,4 @@ -import { DateTime, Effect, Exit, FileSystem, Schedule, Scope, Semaphore } from "effect"; +import { DateTime, Effect, Exit, FileSystem, Option, Schedule, Scope, Semaphore } from "effect"; import { namespaceError, type NamespaceError } from "./Capabilities.ts"; import { acquireLock, isBusy, takeSharedLock } from "./drivers/Sqlite.ts"; @@ -78,3 +78,22 @@ export const pin = ( }), ), ); + +/** + * Runs `attempt` (a non-pinning, best-effort reader like a retirement sweep) only while this + * process holds no live local pin on `lockPath`, under the same per-path permit `pin` itself + * takes. Without this, a sweep's own `acquireLock` and a concurrent first-time `pin` on the same + * path race to open independent connections, and the loser sees "locked by this process" instead + * of either properly queuing or deferring; sharing the permit makes them mutually exclusive, so a + * sweep either runs to completion before a fresh pin ever opens a connection, or correctly finds + * the path already pinned and skips. + */ +export const withoutLocalPin = ( + lockPath: string, + attempt: Effect.Effect, +): Effect.Effect, E, R> => + semaphoreFor(lockPath).withPermits(1)( + Effect.suspend((): Effect.Effect, E, R> => + registry.has(lockPath) ? Effect.succeed(Option.none()) : Effect.map(attempt, Option.some), + ), + ); diff --git a/packages/stack/src/preparation/ArtifactStore.ts b/packages/stack/src/preparation/ArtifactStore.ts index e4ff7504cd..08e6822faf 100644 --- a/packages/stack/src/preparation/ArtifactStore.ts +++ b/packages/stack/src/preparation/ArtifactStore.ts @@ -632,7 +632,9 @@ const discoverGenerations = ( * AND, rechecked while holding that lock, its mtime is older than the retention window. Digest * lock files are never deleted (a stable inode); only the generation directory they guard is. A * per-digest failure (including contention from a live pin or another retirer) is skipped, never - * propagated: this sweep must never block the `prepare`/`use` call it runs alongside. + * propagated: this sweep must never block the `prepare`/`use` call it runs alongside. Each + * digest's own lock attempt shares `Pin`'s per-path permit for that digest, so it can never open a + * second, independent connection alongside a concurrent first-time local `pin` on the same path. */ const retireStaleGenerations = Effect.fn("ArtifactStore.retire")(function* ( fs: FileSystem.FileSystem, @@ -646,18 +648,24 @@ const retireStaleGenerations = Effect.fn("ArtifactStore.retire")(function* ( const generationPath = path.join(keyRoot, digest); const lockPath = path.join(keyRoot, `${digest}.lock`); const stagingRoot = path.join(keyRoot, STAGING_DIR_NAME); - const didRetire = yield* Effect.scoped( - Effect.gen(function* () { - const connection = yield* acquireLock(lockPath, "existing"); - yield* takeExclusiveLock(connection); - const info = yield* fs.stat(lockPath); - const mtime = Option.getOrUndefined(info.mtime); - const now = yield* Clock.currentTimeMillis; - if (mtime === undefined || now - mtime.getTime() <= RETIREMENT_AGE_MILLIS) return false; - yield* relocateToStaging(fs, path, crypto, stagingRoot, generationPath); - return true; - }), - ).pipe(Effect.orElseSucceed(() => false)); + const didRetire = yield* Pin.withoutLocalPin( + lockPath, + Effect.scoped( + Effect.gen(function* () { + const connection = yield* acquireLock(lockPath, "existing"); + yield* takeExclusiveLock(connection); + const info = yield* fs.stat(lockPath); + const mtime = Option.getOrUndefined(info.mtime); + const now = yield* Clock.currentTimeMillis; + if (mtime === undefined || now - mtime.getTime() <= RETIREMENT_AGE_MILLIS) return false; + yield* relocateToStaging(fs, path, crypto, stagingRoot, generationPath); + return true; + }), + ), + ).pipe( + Effect.map(Option.getOrElse(() => false)), + Effect.orElseSucceed(() => false), + ); if (didRetire) retired++; } yield* Effect.annotateCurrentSpan({ "artifact.retired_count": retired }); diff --git a/packages/stack/src/preparation/artifacts.integration.test.ts b/packages/stack/src/preparation/artifacts.integration.test.ts index cecb3699d6..a4473ff2fb 100644 --- a/packages/stack/src/preparation/artifacts.integration.test.ts +++ b/packages/stack/src/preparation/artifacts.integration.test.ts @@ -823,6 +823,102 @@ describe("pins", () => { ), ); + it.live( + "a fresh pin never fails busy against a same-process retirement sweep triggered through the real store", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-pin-sweep-race-", + }); + const key = "database/postgres-pin-sweep-race"; + + const setupStore = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const prepared = yield* setupStore.prepare({ ...request, key }); + // Idle and past the retention window: ahead-of-time `prepare` only pins itself for its + // own duration (see `ArtifactStore.prepare`), so by now nothing in this process still + // pins it, making it a legitimate, un-pinned sweep target. + yield* ageLockFile(fs, prepared.lockPath); + const keyRoot = path.dirname(prepared.path); + + // Gates the sweep right after it opens its own connection on the target's lock path + // (inside `acquireLock`/`takeExclusiveLock`, just before `retireStaleGenerations`' + // `fs.stat` call), holding it open across a real async gap under full test control. + const sweepIsHolding = yield* Deferred.make(); + const releaseSweep = yield* Deferred.make(); + const sweepStore = yield* makeArtifactStore({ + cacheRoot: root, + source: sourceWritingAlt(), + }).pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fs, + stat: (...args: Parameters) => + args[0] === prepared.lockPath + ? Deferred.succeed(sweepIsHolding, undefined).pipe( + Effect.andThen(Deferred.await(releaseSweep)), + Effect.andThen(fs.stat(...args)), + ) + : fs.stat(...args), + }), + ); + + // Gates the fresh `use` right before it calls `Pin.pin`, once it has resolved the same + // generation and confirmed the key directory (the step immediately preceding the pin). + const freshPinApproaching = yield* Deferred.make(); + const letFreshPinProceed = yield* Deferred.make(); + const freshPinStore = yield* makeArtifactStore({ + cacheRoot: root, + source: sourceWriting(), + }).pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fs, + realPath: (...args: Parameters) => + args[0] === keyRoot + ? Deferred.succeed(freshPinApproaching, undefined).pipe( + Effect.andThen(Deferred.await(letFreshPinProceed)), + Effect.andThen(fs.realPath(...args)), + ) + : fs.realPath(...args), + }), + ); + + // A failed assertion below must not leave either fiber parked on its gate forever. + yield* Effect.addFinalizer(() => + Effect.all([ + Deferred.succeed(releaseSweep, undefined), + Deferred.succeed(letFreshPinProceed, undefined), + ]), + ); + + const sweepFiber = yield* sweepStore + .prepare({ ...request, key }) + .pipe(Effect.exit, Effect.forkScoped); + yield* Deferred.await(sweepIsHolding); + + const freshPinFiber = yield* Effect.scoped(freshPinStore.use({ ...request, key })).pipe( + Effect.exit, + Effect.forkScoped, + ); + yield* Deferred.await(freshPinApproaching); + + // Releases the fresh pin into its own `Pin.pin` call first, while the sweep still holds + // the identical path open, then lets the sweep finish: this is the exact collision an + // unsynchronized sweep would lose. + yield* Deferred.succeed(letFreshPinProceed, undefined); + yield* Deferred.succeed(releaseSweep, undefined); + + const sweepExit = yield* Fiber.join(sweepFiber); + const freshPinExit = yield* Fiber.join(freshPinFiber); + const message = (exit: Exit.Exit) => + Exit.isFailure(exit) ? Cause.pretty(exit.cause) : ""; + expect(Exit.isSuccess(sweepExit), message(sweepExit)).toBe(true); + expect(Exit.isSuccess(freshPinExit), message(freshPinExit)).toBe(true); + }), + ), + ); + it.live( "the native launcher holds its own pin independent of the owner, releasing it only once it reports the workload's exit", () => From a9581b5326a0c8adf04da7a4e954b1e9cd66c2ea Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sun, 4 Oct 2026 15:53:02 +0200 Subject: [PATCH 026/142] feat(stack): add a pure lifecycle reducer for wake, sleep, demand and breaker policy --- packages/stack/src/Lifecycle.ts | 1008 ++++++++++++++++ packages/stack/src/Lifecycle.unit.test.ts | 1294 +++++++++++++++++++++ 2 files changed, 2302 insertions(+) create mode 100644 packages/stack/src/Lifecycle.ts create mode 100644 packages/stack/src/Lifecycle.unit.test.ts diff --git a/packages/stack/src/Lifecycle.ts b/packages/stack/src/Lifecycle.ts new file mode 100644 index 0000000000..58cfa84f65 --- /dev/null +++ b/packages/stack/src/Lifecycle.ts @@ -0,0 +1,1008 @@ +import { Data } from "effect"; + +/** Default per-waiter wake budget; a service's spec may override it for known-slow launches. */ +const defaultWaiterBudgetMillis = 120_000; +const waiterCap = 256; +const breakerThreshold = 3; +const breakerBaseCooldownMillis = 30_000; +const breakerMaxCooldownMillis = 300_000; +const breakerStabilityMillis = 30_000; + +/** One member of the owner's composition graph, as the reducer needs to see it. */ +export interface ServiceSpec { + readonly id: string; + readonly activation: "eager" | "lazy"; + readonly prerequisites: ReadonlyArray; + /** Undefined means the service never sleeps on idle. */ + readonly idleMillis?: number; + readonly waiterBudgetMillis?: number; +} + +/** + * The owner's composition graph, with its topological order precomputed, plus each service's + * direct dependents (for the stop/restart guard), full transitive-dependent closure (every + * service that needs it, directly or through another prerequisite, so a wake deep in a stopped + * chain is visible to every ancestor in one pass) and full transitive prerequisite closure (so + * readiness checks see a lost ancestor, not just a direct prerequisite). + */ +export interface LifecycleGraph { + readonly services: ReadonlyMap; + readonly directDependents: ReadonlyMap>; + readonly transitiveDependents: ReadonlyMap>; + readonly prerequisiteClosure: ReadonlyMap>; + readonly order: ReadonlyArray; +} + +export const makeGraph = (specs: ReadonlyArray): LifecycleGraph => { + const services = new Map(specs.map((spec): [string, ServiceSpec] => [spec.id, spec])); + const order: Array = []; + const seen = new Set(); + const prerequisiteClosure = new Map>(); + const closureOf = (id: string): ReadonlySet => { + const cached = prerequisiteClosure.get(id); + if (cached !== undefined) return cached; + const closure = new Set(); + for (const prerequisite of services.get(id)?.prerequisites ?? []) { + closure.add(prerequisite); + for (const transitive of closureOf(prerequisite)) closure.add(transitive); + } + prerequisiteClosure.set(id, closure); + return closure; + }; + const visit = (id: string): void => { + if (seen.has(id)) return; + seen.add(id); + for (const prerequisite of services.get(id)?.prerequisites ?? []) visit(prerequisite); + order.push(id); + }; + const directDependents = new Map>(); + const transitiveDependents = new Map>(); + for (const id of services.keys()) { + visit(id); + for (const prerequisite of services.get(id)?.prerequisites ?? []) { + const list = directDependents.get(prerequisite) ?? []; + list.push(id); + directDependents.set(prerequisite, list); + } + for (const prerequisite of closureOf(id)) { + const list = transitiveDependents.get(prerequisite) ?? new Set(); + list.add(id); + transitiveDependents.set(prerequisite, list); + } + } + return { services, directDependents, transitiveDependents, prerequisiteClosure, order }; +}; + +type Intent = "eager" | "lazy" | "stopped"; + +/** A launch's progress, named so a waiter's budget error can report where it's still blocked. */ +type Stage = "preparing" | "artifacts" | "launching"; + +/** + * A service's point in its launch/stop cycle. `Running.ready` tracks the live health signal + * separately from the phase itself: a readiness failure is not terminal, only a launch + * failure or an exit is. + */ +type Phase = Data.TaggedEnum<{ + Stopped: {}; + Starting: { readonly generation: number; readonly stage: Stage }; + Running: { readonly generation: number; readonly ready: boolean }; + Stopping: { readonly generation: number }; + Failed: { readonly generation: number; readonly cause: unknown }; +}>; +const Phase = Data.taggedEnum(); + +interface BreakerState { + readonly consecutiveFailures: number; + readonly openUntil: number | undefined; + /** The cooldown this breaker will use next time it opens; doubles on each repeat open. */ + readonly cooldownMillis: number; + readonly lastCause: unknown; + /** The generation and timestamp its stability reset is counting from; cleared by a readiness loss or an exit. */ + readonly stableGeneration: number | undefined; + readonly stableSince: number | undefined; +} + +const initialBreaker: BreakerState = { + consecutiveFailures: 0, + openUntil: undefined, + cooldownMillis: breakerBaseCooldownMillis, + lastCause: undefined, + stableGeneration: undefined, + stableSince: undefined, +}; + +interface Waiter { + readonly id: number; + readonly deadline: number; + /** False for an acquisition (such as the Functions inspector) that bypasses target readiness. */ + readonly requireReady: boolean; +} + +export interface ServiceState { + readonly intent: Intent; + readonly phase: Phase; + readonly leases: number; + readonly lastActivity: number; + readonly idleEpoch: number; + /** The epoch an outstanding idle timer was armed with, or undefined while none is armed. */ + readonly idleArmedEpoch: number | undefined; + readonly breaker: BreakerState; + readonly waiters: ReadonlyMap; + /** Set by an explicit start or restart so the next launch attempt proceeds without demand. */ + readonly relaunchForced: boolean; + /** Blocks every launch while a storage operation owns the stopped service. */ + readonly storageReserved: boolean; +} + +const initialService = (spec: ServiceSpec, now: number): ServiceState => ({ + intent: spec.activation, + phase: Phase.Stopped(), + leases: 0, + lastActivity: now, + idleEpoch: 0, + idleArmedEpoch: undefined, + breaker: initialBreaker, + waiters: new Map(), + relaunchForced: false, + storageReserved: false, +}); + +export interface LifecycleState { + readonly graph: LifecycleGraph; + readonly services: ReadonlyMap; + /** + * The next generation to assign per service, kept outside `services` and never pruned: a + * service removed and re-added by a `GraphUpdated` still can't reuse a generation number. + */ + readonly generationCounters: ReadonlyMap; +} + +export const initialState = (graph: LifecycleGraph, now: number): LifecycleState => ({ + graph, + services: new Map( + [...graph.services.values()].map((spec): [string, ServiceState] => [ + spec.id, + initialService(spec, now), + ]), + ), + generationCounters: new Map([...graph.services.keys()].map((id) => [id, 1])), +}); + +export type LifecycleEvent = Data.TaggedEnum<{ + /** `requireReady: false` lets an acquisition (the Functions inspector) bypass target readiness. */ + ConnectionOpened: { + readonly id: string; + readonly waiterId: number; + readonly requireReady: boolean; + }; + ConnectionClosed: { readonly id: string }; + /** A live session exists but hasn't passed its first health check; admits `requireReady: false`. */ + SessionAvailable: { readonly id: string; readonly generation: number }; + LaunchSucceeded: { readonly id: string; readonly generation: number }; + LaunchFailed: { readonly id: string; readonly generation: number; readonly cause: unknown }; + Exited: { + readonly id: string; + readonly generation: number; + readonly cause: unknown; + readonly requested: boolean; + }; + IdleElapsed: { readonly id: string; readonly generation: number; readonly epoch: number }; + ReadinessLost: { readonly id: string; readonly generation: number; readonly cause: unknown }; + ReadinessRecovered: { readonly id: string; readonly generation: number }; + StageChanged: { readonly id: string; readonly generation: number; readonly stage: Stage }; + /** Fired by the runtime once a breaker's cooldown elapses; stale against the current breaker is a no-op. */ + CooldownElapsed: { readonly id: string; readonly openUntil: number }; + StartRequested: { readonly id: string }; + StopRequested: { readonly id: string }; + RestartRequested: { readonly id: string }; + StorageReserved: { readonly id: string }; + StorageReleased: { readonly id: string }; + WaiterExpired: { readonly id: string; readonly waiterId: number }; + WaiterCancelled: { readonly id: string; readonly waiterId: number }; + /** Atomically swaps the graph while preserving every continuing service's generation and epoch. */ + GraphUpdated: { readonly graph: LifecycleGraph }; +}>; +export const LifecycleEvent = Data.taggedEnum(); + +export type LifecycleCommand = Data.TaggedEnum<{ + Launch: { readonly id: string; readonly generation: number }; + Stop: { readonly id: string; readonly generation: number }; + ArmIdleTimer: { + readonly id: string; + readonly generation: number; + readonly epoch: number; + readonly delayMillis: number; + }; + ArmWaiterTimeout: { readonly id: string; readonly waiterId: number; readonly deadline: number }; + /** Schedules the `CooldownElapsed` event once the breaker's cooldown elapses. */ + ArmCooldownTimer: { + readonly id: string; + readonly openUntil: number; + readonly delayMillis: number; + }; + /** Re-run the live session's readiness probe after it lost readiness without exiting. */ + Reprobe: { readonly id: string; readonly generation: number }; + AdmitConnection: { readonly id: string; readonly waiterId: number }; + FailConnection: { + readonly id: string; + readonly waiterId: number; + readonly message: string; + readonly cause: unknown; + }; + /** An explicit stop, restart, storage reservation or graph update the reducer refused to admit. */ + RequestRejected: { + readonly id: string; + readonly operation: "stop" | "restart" | "storage" | "graph"; + readonly message: string; + }; +}>; +export const LifecycleCommand = Data.taggedEnum(); + +const generationOf = (phase: Phase): number | undefined => + Phase.$match(phase, { + Stopped: () => undefined, + Starting: (p) => p.generation, + Running: (p) => p.generation, + Stopping: (p) => p.generation, + Failed: (p) => p.generation, + }); + +const isUpOrWindingDown = (phase: Phase): boolean => + phase._tag === "Starting" || phase._tag === "Running" || phase._tag === "Stopping"; + +const isReady = (phase: Phase): boolean => phase._tag === "Running" && phase.ready; +const isSessionAvailable = (phase: Phase): boolean => phase._tag === "Running"; + +const ownsDemand = (service: ServiceState): boolean => + service.intent === "eager" || + service.leases > 0 || + service.waiters.size > 0 || + service.relaunchForced; + +/** + * A dependent, anywhere in the transitive closure, that an explicit stop or restart of its + * prerequisite would strand: one with live or retained demand, one still winding down, or one + * merely wake-armed (lazy or eager, not explicitly stopped) and so able to need it again at any + * moment. Internal idle sleep uses its own, narrower demand rule and is unaffected by this. + */ +const dependentIsActive = (service: ServiceState): boolean => + service.intent !== "stopped" || ownsDemand(service) || isUpOrWindingDown(service.phase); + +/** A service active enough that a `GraphUpdated` removing or changing it would drop live or retained work. */ +const isActiveForGraphUpdate = (service: ServiceState): boolean => + isUpOrWindingDown(service.phase) || + service.leases > 0 || + service.waiters.size > 0 || + service.storageReserved || + service.relaunchForced; + +/** Whether two specs for the same id describe the same service, ignoring prerequisite order. */ +const specsEqual = (a: ServiceSpec, b: ServiceSpec): boolean => + a.activation === b.activation && + a.idleMillis === b.idleMillis && + a.waiterBudgetMillis === b.waiterBudgetMillis && + a.prerequisites.length === b.prerequisites.length && + a.prerequisites.every((prerequisite) => b.prerequisites.includes(prerequisite)); + +/** + * A service has demand when it owns work itself (including an explicit start/restart that hasn't + * launched yet), or a transitive dependent does, or a transitive dependent is admitted, starting + * or running (held until its exit is confirmed). The dependent closure is checked directly (not + * just the immediate one) so a wake deep in a fully stopped chain is visible to every prerequisite + * in the same pass. + */ +const hasDemand = (state: LifecycleState, id: string): boolean => { + const service = state.services.get(id); + if (service === undefined) return false; + if (ownsDemand(service)) return true; + for (const dependent of state.graph.transitiveDependents.get(id) ?? []) { + const dependentState = state.services.get(dependent); + if ( + dependentState !== undefined && + (ownsDemand(dependentState) || isUpOrWindingDown(dependentState.phase)) + ) + return true; + } + return false; +}; + +/** Every prerequisite in the full transitive closure must itself be ready, not just direct ones. */ +const prerequisitesSatisfied = (state: LifecycleState, id: string): boolean => { + for (const prerequisite of state.graph.prerequisiteClosure.get(id) ?? []) { + const prerequisiteState = state.services.get(prerequisite); + if (prerequisiteState === undefined || !isReady(prerequisiteState.phase)) return false; + } + return true; +}; + +/** Effective readiness for admission: the target's own health and its prerequisite closure. */ +const admissionReady = (state: LifecycleState, id: string): boolean => { + const service = state.services.get(id); + return service !== undefined && isReady(service.phase) && prerequisitesSatisfied(state, id); +}; + +/** A live session, bypassing the target's own health check, still behind its prerequisite closure. */ +const sessionReady = (state: LifecycleState, id: string): boolean => { + const service = state.services.get(id); + return ( + service !== undefined && isSessionAvailable(service.phase) && prerequisitesSatisfied(state, id) + ); +}; + +const isStabilized = (breaker: BreakerState, phase: Phase, now: number): boolean => + breaker.stableGeneration !== undefined && + breaker.stableGeneration === generationOf(phase) && + breaker.stableSince !== undefined && + now - breaker.stableSince >= breakerStabilityMillis; + +/** Bakes in a stability reset that has already been earned before its evidence is discarded. */ +const materializeStability = (breaker: BreakerState, phase: Phase, now: number): BreakerState => + isStabilized(breaker, phase, now) + ? { + ...breaker, + consecutiveFailures: 0, + cooldownMillis: breakerBaseCooldownMillis, + stableGeneration: undefined, + stableSince: undefined, + } + : breaker; + +/** + * Whether the breaker is currently blocking admission. This is a plain flag, not a `now` + * comparison: only the dedicated `CooldownElapsed` event clears it, so a stale or unrelated event + * processed after the cooldown has elapsed on the wall clock can never itself reopen admission. + */ +const isBreakerOpen = (breaker: BreakerState): boolean => breaker.openUntil !== undefined; + +/** Names what a pending waiter is still blocked on, for a budget-expiry error. */ +const blockingStage = (state: LifecycleState, id: string): string => { + const service = state.services.get(id); + if (service === undefined) return `${id} is unknown`; + for (const prerequisite of state.graph.prerequisiteClosure.get(id) ?? []) { + const prerequisiteState = state.services.get(prerequisite); + if (prerequisiteState === undefined || !isReady(prerequisiteState.phase)) + return `prerequisite ${prerequisite}`; + } + if (service.phase._tag === "Starting") return `${id} to finish ${service.phase.stage}`; + if (service.phase._tag !== "Running") return `${id} to start`; + if (!service.phase.ready) return `${id} to become healthy`; + return `${id}`; +}; + +const setService = ( + state: LifecycleState, + id: string, + update: (service: ServiceState) => ServiceState, +): LifecycleState => { + const service = state.services.get(id); + if (service === undefined) return state; + return { ...state, services: new Map(state.services).set(id, update(service)) }; +}; + +/** Fails every pending waiter of a service with one cause and clears its waiter queue. */ +const failAllWaiters = ( + service: ServiceState, + id: string, + message: string, + cause: unknown, + commands: Array, +): ServiceState => { + for (const waiter of service.waiters.values()) + commands.push(LifecycleCommand.FailConnection({ id, waiterId: waiter.id, message, cause })); + return { ...service, waiters: new Map() }; +}; + +/** The transitive dependents currently active enough to block a stop or restart of a prerequisite. */ +const activeDependents = (state: LifecycleState, id: string): ReadonlyArray => + [...(state.graph.transitiveDependents.get(id) ?? [])].filter((dependentId) => { + const dependentState = state.services.get(dependentId); + return dependentState !== undefined && dependentIsActive(dependentState); + }); + +/** Re-arms every prerequisite an explicit start or restart depends on, clearing a stopped intent. */ +const rearmPrerequisites = (state: LifecycleState, id: string): LifecycleState => { + let next = state; + for (const prerequisite of state.graph.prerequisiteClosure.get(id) ?? []) { + const prerequisiteSpec = state.graph.services.get(prerequisite); + if (prerequisiteSpec === undefined) continue; + next = setService(next, prerequisite, (s) => + s.intent === "stopped" ? { ...s, intent: prerequisiteSpec.activation } : s, + ); + } + return next; +}; + +/** Shared breaker/phase bookkeeping for a launch failure or an unrequested exit. */ +const recordFailure = ( + state: LifecycleState, + id: string, + generation: number, + cause: unknown, + now: number, + commands: Array, +): LifecycleState => + setService(state, id, (service) => { + const materialized = materializeStability(service.breaker, service.phase, now); + const consecutiveFailures = materialized.consecutiveFailures + 1; + const opens = consecutiveFailures >= breakerThreshold; + let breaker: BreakerState; + if (opens) { + const openUntil = now + materialized.cooldownMillis; + commands.push( + LifecycleCommand.ArmCooldownTimer({ + id, + openUntil, + delayMillis: materialized.cooldownMillis, + }), + ); + breaker = { + consecutiveFailures, + openUntil, + cooldownMillis: Math.min(breakerMaxCooldownMillis, materialized.cooldownMillis * 2), + lastCause: cause, + stableGeneration: undefined, + stableSince: undefined, + }; + } else { + breaker = { + consecutiveFailures, + openUntil: undefined, + cooldownMillis: materialized.cooldownMillis, + lastCause: cause, + stableGeneration: undefined, + stableSince: undefined, + }; + } + const failed = failAllWaiters(service, id, `${id} failed`, cause, commands); + return { + ...failed, + phase: Phase.Failed({ generation, cause }), + breaker, + idleArmedEpoch: undefined, + relaunchForced: false, + }; + }); + +const applyEvent = ( + state: LifecycleState, + event: LifecycleEvent, + now: number, +): readonly [LifecycleState, ReadonlyArray] => { + const commands: Array = []; + + switch (event._tag) { + case "ConnectionOpened": { + const { id, waiterId, requireReady } = event; + const service = state.services.get(id); + const spec = state.graph.services.get(id); + if (service === undefined || spec === undefined) break; + if (service.intent === "stopped") { + commands.push( + LifecycleCommand.FailConnection({ + id, + waiterId, + message: `${id} is explicitly stopped`, + cause: undefined, + }), + ); + break; + } + const eligibleNow = requireReady ? admissionReady(state, id) : sessionReady(state, id); + if (eligibleNow) { + return [ + setService(state, id, (s) => ({ ...s, leases: s.leases + 1, lastActivity: now })), + [LifecycleCommand.AdmitConnection({ id, waiterId })], + ]; + } + if (isBreakerOpen(service.breaker)) { + commands.push( + LifecycleCommand.FailConnection({ + id, + waiterId, + message: `${id} circuit breaker is open`, + cause: service.breaker.lastCause, + }), + ); + break; + } + if (service.waiters.size >= waiterCap) { + commands.push( + LifecycleCommand.FailConnection({ + id, + waiterId, + message: `${id} has too many waiters`, + cause: undefined, + }), + ); + break; + } + const deadline = now + (spec.waiterBudgetMillis ?? defaultWaiterBudgetMillis); + commands.push(LifecycleCommand.ArmWaiterTimeout({ id, waiterId, deadline })); + return [ + setService(state, id, (s) => ({ + ...s, + lastActivity: now, + waiters: new Map(s.waiters).set(waiterId, { id: waiterId, deadline, requireReady }), + })), + commands, + ]; + } + case "ConnectionClosed": { + return [ + setService(state, event.id, (s) => ({ + ...s, + leases: Math.max(0, s.leases - 1), + lastActivity: now, + })), + commands, + ]; + } + case "SessionAvailable": { + const { id, generation } = event; + const service = state.services.get(id); + if ( + service === undefined || + service.phase._tag !== "Starting" || + service.phase.generation !== generation + ) + break; + return [ + setService(state, id, (s) => ({ + ...s, + phase: Phase.Running({ generation, ready: false }), + })), + commands, + ]; + } + case "LaunchSucceeded": { + const { id, generation } = event; + const service = state.services.get(id); + if (service === undefined || generationOf(service.phase) !== generation) break; + // Accepted either straight from `Starting` or after a `SessionAvailable` left it unready. + const fromStarting = service.phase._tag === "Starting"; + const fromUnreadySession = service.phase._tag === "Running" && !service.phase.ready; + if (!fromStarting && !fromUnreadySession) break; + return [ + setService(state, id, (s) => ({ + ...s, + phase: Phase.Running({ generation, ready: true }), + breaker: { ...s.breaker, stableGeneration: generation, stableSince: now }, + })), + commands, + ]; + } + case "LaunchFailed": { + const { id, generation, cause } = event; + const service = state.services.get(id); + if ( + service === undefined || + service.phase._tag !== "Starting" || + service.phase.generation !== generation + ) + break; + return [recordFailure(state, id, generation, cause, now, commands), commands]; + } + case "Exited": { + const { id, generation, cause, requested } = event; + const service = state.services.get(id); + if (service === undefined || generationOf(service.phase) !== generation) break; + if (service.phase._tag === "Stopping" || requested) { + return [ + setService(state, id, (s) => ({ + ...s, + phase: Phase.Stopped(), + idleArmedEpoch: undefined, + })), + commands, + ]; + } + if (service.phase._tag === "Running" || service.phase._tag === "Starting") + return [recordFailure(state, id, generation, cause, now, commands), commands]; + break; + } + case "IdleElapsed": { + const { id, generation, epoch } = event; + const service = state.services.get(id); + if ( + service === undefined || + service.phase._tag !== "Running" || + service.phase.generation !== generation || + service.idleArmedEpoch !== epoch || + hasDemand(state, id) + ) + break; + commands.push(LifecycleCommand.Stop({ id, generation })); + return [ + setService(state, id, (s) => ({ + ...s, + phase: Phase.Stopping({ generation }), + idleArmedEpoch: undefined, + // Bake in any 30s stability the just-ending generation already earned, since the next + // failure to check it will compare against a different, unrelated generation. + breaker: materializeStability(s.breaker, s.phase, now), + })), + commands, + ]; + } + case "ReadinessLost": { + const { id, generation } = event; + const service = state.services.get(id); + if ( + service === undefined || + service.phase._tag !== "Running" || + service.phase.generation !== generation + ) + break; + commands.push(LifecycleCommand.Reprobe({ id, generation })); + return [ + setService(state, id, (s) => ({ + ...s, + phase: Phase.Running({ generation, ready: false }), + breaker: { + ...materializeStability(s.breaker, s.phase, now), + stableGeneration: undefined, + stableSince: undefined, + }, + })), + commands, + ]; + } + case "ReadinessRecovered": { + const { id, generation } = event; + const service = state.services.get(id); + if ( + service === undefined || + service.phase._tag !== "Running" || + service.phase.generation !== generation + ) + break; + return [ + setService(state, id, (s) => ({ + ...s, + phase: Phase.Running({ generation, ready: true }), + breaker: { ...s.breaker, stableGeneration: generation, stableSince: now }, + })), + commands, + ]; + } + case "StageChanged": { + const { id, generation, stage } = event; + const service = state.services.get(id); + if ( + service === undefined || + service.phase._tag !== "Starting" || + service.phase.generation !== generation + ) + break; + return [ + setService(state, id, (s) => ({ ...s, phase: Phase.Starting({ generation, stage }) })), + commands, + ]; + } + case "CooldownElapsed": { + const { id, openUntil } = event; + const service = state.services.get(id); + if (service === undefined || service.breaker.openUntil !== openUntil) break; + return [ + setService(state, id, (s) => ({ ...s, breaker: { ...s.breaker, openUntil: undefined } })), + commands, + ]; + } + case "StartRequested": { + const { id } = event; + const service = state.services.get(id); + const spec = state.graph.services.get(id); + if (service === undefined || spec === undefined) break; + // A service already `Running` has nothing left to force, and forcing it here would wrongly + // survive into a later, unrelated idle cycle. One already `Stopping` is different: that + // generation's end is imminent and this request's demand must carry through to the next one + // (its own exit confirmation is the only thing that will ever consume this flag). + const forced = + service.phase._tag === "Stopped" || + service.phase._tag === "Failed" || + service.phase._tag === "Stopping"; + const updated = setService(state, id, (s) => ({ + ...s, + intent: spec.activation, + breaker: initialBreaker, + relaunchForced: s.relaunchForced || forced, + })); + return [rearmPrerequisites(updated, id), commands]; + } + case "StopRequested": { + const { id } = event; + const service = state.services.get(id); + if (service === undefined) break; + // Checked independently of whether this service itself needs a `Stop` command: a `Failed` + // target with an open breaker must still protect a dependent's retained demand. + const blockers = activeDependents(state, id); + if (blockers.length > 0) { + commands.push( + LifecycleCommand.RequestRejected({ + id, + operation: "stop", + message: `${id} has active dependents: ${blockers.join(", ")}`, + }), + ); + break; + } + const winding = isUpOrWindingDown(service.phase) && service.phase._tag !== "Stopping"; + const failed = failAllWaiters( + service, + id, + `${id} was explicitly stopped`, + undefined, + commands, + ); + const generation = generationOf(failed.phase); + if (winding && generation !== undefined) + commands.push(LifecycleCommand.Stop({ id, generation })); + const phase = + winding && generation !== undefined + ? Phase.Stopping({ generation }) + : failed.phase._tag === "Failed" + ? Phase.Stopped() + : failed.phase; + return [ + setService(state, id, () => ({ + ...failed, + intent: "stopped", + relaunchForced: false, + phase, + })), + commands, + ]; + } + case "RestartRequested": { + const { id } = event; + const service = state.services.get(id); + const spec = state.graph.services.get(id); + if (service === undefined || spec === undefined) break; + const blockers = activeDependents(state, id); + if (blockers.length > 0) { + commands.push( + LifecycleCommand.RequestRejected({ + id, + operation: "restart", + message: `${id} has active dependents: ${blockers.join(", ")}`, + }), + ); + break; + } + const winding = isUpOrWindingDown(service.phase) && service.phase._tag !== "Stopping"; + const generation = generationOf(service.phase); + if (winding && generation !== undefined) + commands.push(LifecycleCommand.Stop({ id, generation })); + const updated = setService(state, id, (s) => ({ + ...s, + intent: spec.activation, + breaker: initialBreaker, + relaunchForced: true, + phase: winding && generation !== undefined ? Phase.Stopping({ generation }) : s.phase, + })); + return [rearmPrerequisites(updated, id), commands]; + } + case "StorageReserved": { + const { id } = event; + const service = state.services.get(id); + if (service === undefined) break; + if ( + service.phase._tag !== "Stopped" || + service.intent !== "stopped" || + service.storageReserved + ) { + commands.push( + LifecycleCommand.RequestRejected({ + id, + operation: "storage", + message: `${id} must be stopped before a storage operation`, + }), + ); + break; + } + return [setService(state, id, (s) => ({ ...s, storageReserved: true })), commands]; + } + case "StorageReleased": { + return [setService(state, event.id, (s) => ({ ...s, storageReserved: false })), commands]; + } + case "WaiterExpired": { + const { id, waiterId } = event; + const service = state.services.get(id); + const waiter = service?.waiters.get(waiterId); + if (service === undefined || waiter === undefined) break; + commands.push( + LifecycleCommand.FailConnection({ + id, + waiterId, + message: `${id} wake budget exceeded while waiting for ${blockingStage(state, id)}`, + cause: undefined, + }), + ); + return [ + setService(state, id, (s) => { + const waiters = new Map(s.waiters); + waiters.delete(waiterId); + return { ...s, waiters }; + }), + commands, + ]; + } + case "WaiterCancelled": { + const { id, waiterId } = event; + return [ + setService(state, id, (s) => { + if (!s.waiters.has(waiterId)) return s; + const waiters = new Map(s.waiters); + waiters.delete(waiterId); + return { ...s, waiters }; + }), + commands, + ]; + } + case "GraphUpdated": { + const { graph } = event; + const removedIds = [...state.graph.services.keys()].filter((id) => !graph.services.has(id)); + const changedIds = [...state.graph.services.keys()].filter((id) => { + const previousSpec = state.graph.services.get(id); + const nextSpec = graph.services.get(id); + return ( + previousSpec !== undefined && + nextSpec !== undefined && + !specsEqual(previousSpec, nextSpec) + ); + }); + const blockedIds = [...removedIds, ...changedIds].filter((id) => { + const service = state.services.get(id); + return service !== undefined && isActiveForGraphUpdate(service); + }); + if (blockedIds.length > 0) { + for (const id of blockedIds) + commands.push( + LifecycleCommand.RequestRejected({ + id, + operation: "graph", + message: `${id} is active and can't be removed or changed by a graph update`, + }), + ); + break; + } + const services = new Map(); + const generationCounters = new Map(state.generationCounters); + for (const spec of graph.services.values()) { + const existing = state.services.get(spec.id); + services.set(spec.id, existing ?? initialService(spec, now)); + if (!generationCounters.has(spec.id)) generationCounters.set(spec.id, 1); + } + return [{ graph, services, generationCounters }, commands]; + } + } + + return [state, commands]; +}; + +/** + * Cascades launches, idle arming and waiter resolution across the graph after one event. A + * single topological pass suffices: a launch started here only reaches `Running` on a later + * event, so a dependent several hops away naturally waits for its own future turn. Every + * condition here is a function of state alone, never of `now` against a stored deadline, so + * replaying this pass for a stale or unrelated event can never surface a new launch or admission + * on its own; only the dedicated time-driven events (`CooldownElapsed`, `IdleElapsed`, + * `WaiterExpired`) do that, and the one exception — expiring an already-due waiter instead of + * admitting it — only fires from within this same pass's own admission step. + */ +const settle = ( + state: LifecycleState, + now: number, + commands: ReadonlyArray, +): readonly [LifecycleState, ReadonlyArray] => { + let next = state; + const emitted: Array = [...commands]; + + for (const id of next.graph.order) { + const spec = next.graph.services.get(id); + const service = next.services.get(id); + if (spec === undefined || service === undefined) continue; + + if (service.phase._tag === "Stopped" || service.phase._tag === "Failed") { + if (service.intent === "stopped") continue; + if (service.storageReserved) continue; + if (isBreakerOpen(service.breaker)) continue; + if (!prerequisitesSatisfied(next, id)) continue; + if (!hasDemand(next, id) && !service.relaunchForced) continue; + const generation = next.generationCounters.get(id) ?? 1; + emitted.push(LifecycleCommand.Launch({ id, generation })); + next = { + ...setService(next, id, (s) => ({ + ...s, + phase: Phase.Starting({ generation, stage: "preparing" }), + relaunchForced: false, + })), + generationCounters: new Map(next.generationCounters).set(id, generation + 1), + }; + continue; + } + + if (service.phase._tag === "Running") { + const generation = service.phase.generation; + if (prerequisitesSatisfied(next, id) && service.waiters.size > 0) { + const ready = service.phase.ready; + const admitted: Array = []; + const expired: Array = []; + for (const waiter of service.waiters.values()) { + if (waiter.requireReady && !ready) continue; + if (now >= waiter.deadline) expired.push(waiter.id); + else admitted.push(waiter.id); + } + if (admitted.length > 0 || expired.length > 0) { + for (const waiterId of admitted) + emitted.push(LifecycleCommand.AdmitConnection({ id, waiterId })); + for (const waiterId of expired) + emitted.push( + LifecycleCommand.FailConnection({ + id, + waiterId, + message: `${id} wake budget exceeded while waiting for ${blockingStage(next, id)}`, + cause: undefined, + }), + ); + next = setService(next, id, (s) => { + const waiters = new Map(s.waiters); + for (const waiterId of admitted) waiters.delete(waiterId); + for (const waiterId of expired) waiters.delete(waiterId); + return { ...s, leases: s.leases + admitted.length, waiters, lastActivity: now }; + }); + } + } + const settled = next.services.get(id); + if (settled === undefined) continue; + const idles = + settled.intent === "lazy" && + spec.idleMillis !== undefined && + settled.phase._tag === "Running"; + if (idles && settled.phase._tag === "Running") { + const demand = hasDemand(next, id); + if (demand && settled.idleArmedEpoch !== undefined) { + // Demand returning invalidates any outstanding timer regardless of readiness: an idle + // arm/clear decision must never be stranded by a readiness loss that follows it, or the + // service could never become idle-eligible again once readiness recovers. + next = setService(next, id, (s) => ({ ...s, idleArmedEpoch: undefined })); + } else if (!demand && settled.phase.ready && settled.idleArmedEpoch === undefined) { + const epoch = settled.idleEpoch + 1; + emitted.push( + LifecycleCommand.ArmIdleTimer({ + id, + generation, + epoch, + delayMillis: spec.idleMillis ?? 0, + }), + ); + next = setService(next, id, (s) => ({ ...s, idleEpoch: epoch, idleArmedEpoch: epoch })); + } + } + } + } + + return [next, emitted]; +}; + +/** The one pure `(state, event, now) -> (state, commands)` step for an owner's whole graph. */ +export const reduce = ( + state: LifecycleState, + event: LifecycleEvent, + now: number, +): readonly [LifecycleState, ReadonlyArray] => { + const [applied, directCommands] = applyEvent(state, event, now); + return settle(applied, now, directCommands); +}; + +/** A storage operation is admitted only while a service is stopped with no wake armed. */ +export const canRunStorage = (state: LifecycleState, id: string): boolean => { + const service = state.services.get(id); + return ( + service !== undefined && + service.intent === "stopped" && + service.phase._tag === "Stopped" && + !service.storageReserved + ); +}; diff --git a/packages/stack/src/Lifecycle.unit.test.ts b/packages/stack/src/Lifecycle.unit.test.ts new file mode 100644 index 0000000000..e00d06f633 --- /dev/null +++ b/packages/stack/src/Lifecycle.unit.test.ts @@ -0,0 +1,1294 @@ +import { describe, expect, it } from "@effect/vitest"; +import { + canRunStorage, + initialState, + LifecycleEvent, + makeGraph, + reduce, + type LifecycleCommand as Command, + type LifecycleState, + type ServiceSpec, +} from "./Lifecycle.ts"; + +const lazy = (id: string, overrides: Partial = {}): ServiceSpec => ({ + id, + activation: "lazy", + prerequisites: [], + idleMillis: 1_000, + ...overrides, +}); + +const open = (id: string, waiterId: number, requireReady = true) => + LifecycleEvent.ConnectionOpened({ id, waiterId, requireReady }); + +/** Drives a sequence of events through one lifecycle state, returning each step's commands. */ +const run = ( + initial: LifecycleState, + steps: ReadonlyArray<{ event: LifecycleEvent; now: number }>, +) => { + let state = initial; + const commandsByStep: Array> = []; + for (const { event, now } of steps) { + const [next, commands] = reduce(state, event, now); + state = next; + commandsByStep.push(commands); + } + return { state, commandsByStep }; +}; + +const tagsOf = (commands: ReadonlyArray | undefined) => + (commands ?? []).map((command) => command._tag); + +/** Narrows a service's current phase to `Starting` and returns its generation, or throws. */ +const startingGeneration = (state: LifecycleState, id: string): number => { + const phase = state.services.get(id)?.phase; + if (phase?._tag !== "Starting") throw new Error(`Expected ${id} to be starting`); + return phase.generation; +}; + +/** Reads a service's current generation from any non-`Stopped` phase, or throws. */ +const currentGeneration = (state: LifecycleState, id: string): number => { + const phase = state.services.get(id)?.phase; + if (phase === undefined || phase._tag === "Stopped") + throw new Error(`Expected ${id} to have a generation`); + return phase.generation; +}; + +/** Indexed access with `noUncheckedIndexedAccess`; throws with a clear message if out of bounds. */ +const at = (array: ReadonlyArray, index: number): T => { + const value = array[index]; + if (value === undefined) throw new Error(`Expected an element at index ${index}`); + return value; +}; + +/** Finds one command by tag, or throws; avoids a non-null assertion at call sites. */ +const commandTagged = ( + commands: ReadonlyArray, + tag: Tag, +): Extract => { + const found = commands.find( + (command): command is Extract => command._tag === tag, + ); + if (found === undefined) throw new Error(`Expected a ${tag} command`); + return found; +}; + +describe("Lifecycle reducer", () => { + it("wakes a lazy service on first traffic, admits the waiter once ready, and sleeps after idle", () => { + const graph = makeGraph([lazy("api")]); + const { commandsByStep, state } = run(initialState(graph, 0), [ + { event: open("api", 1), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, + { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 20 }, + ]); + + expect(tagsOf(commandsByStep[0])).toEqual(["ArmWaiterTimeout", "Launch"]); + expect(tagsOf(commandsByStep[1])).toEqual(["AdmitConnection"]); + expect(tagsOf(commandsByStep[2])).toEqual(["ArmIdleTimer"]); + const armed = at(at(commandsByStep, 2), 0) as Extract; + expect(armed).toMatchObject({ id: "api", generation: 1, epoch: 1, delayMillis: 1_000 }); + expect(state.services.get("api")?.phase).toEqual({ + _tag: "Running", + generation: 1, + ready: true, + }); + }); + + it("ignores a launch outcome, an idle elapse and an exit tagged with a stale generation", () => { + const graph = makeGraph([lazy("api")]); + const { state, commandsByStep } = run(initialState(graph, 0), [ + { event: open("api", 1), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 99 }), now: 10 }, + { event: LifecycleEvent.IdleElapsed({ id: "api", generation: 1, epoch: 99 }), now: 20 }, + { + event: LifecycleEvent.Exited({ + id: "api", + generation: 99, + cause: "boom", + requested: false, + }), + now: 30, + }, + ]); + + expect(commandsByStep[1]).toEqual([]); + expect(commandsByStep[2]).toEqual([]); + expect(commandsByStep[3]).toEqual([]); + expect(state.services.get("api")?.phase).toEqual({ + _tag: "Starting", + generation: 1, + stage: "preparing", + }); + }); + + it("ignores an idle elapse tagged with a stale epoch even while a newer timer is armed with zero demand", () => { + const graph = makeGraph([lazy("api")]); + const { state } = run(initialState(graph, 0), [ + { event: open("api", 1), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, + { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 20 }, // arms epoch 1 + { event: open("api", 2), now: 30 }, // demand returns: epoch 1's timer goes stale + { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 40 }, // arms epoch 2 + ]); + expect(state.services.get("api")?.idleArmedEpoch).toBe(2); + + const stale = run(state, [ + { event: LifecycleEvent.IdleElapsed({ id: "api", generation: 1, epoch: 1 }), now: 1_040 }, + ]); + expect(stale.commandsByStep[0]).toEqual([]); + expect(stale.state.services.get("api")?.phase._tag).toBe("Running"); + + const current = run(stale.state, [ + { event: LifecycleEvent.IdleElapsed({ id: "api", generation: 1, epoch: 2 }), now: 1_041 }, + ]); + expect(tagsOf(current.commandsByStep[0])).toEqual(["Stop"]); + }); + + it("keeps a service running when traffic is admitted before the idle commit", () => { + const graph = makeGraph([lazy("api")]); + const { state, commandsByStep } = run(initialState(graph, 0), [ + { event: open("api", 1), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, + { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 20 }, + // Traffic arrives while the timer is armed but before it fires: this prevents the sleep. + { event: open("api", 2), now: 500 }, + { event: LifecycleEvent.IdleElapsed({ id: "api", generation: 1, epoch: 1 }), now: 1_000 }, + ]); + + expect(tagsOf(commandsByStep[3])).toEqual(["AdmitConnection"]); + expect(commandsByStep[4]).toEqual([]); + expect(state.services.get("api")?.phase._tag).toBe("Running"); + expect(state.services.get("api")?.leases).toBe(1); + }); + + it("holds traffic arriving after a committed sleep for the confirmed stop and the next generation, without failing it", () => { + const graph = makeGraph([lazy("api")]); + const { state, commandsByStep } = run(initialState(graph, 0), [ + { event: open("api", 1), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, + { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 20 }, + // The idle timer commits to stopping before any new traffic arrives. + { event: LifecycleEvent.IdleElapsed({ id: "api", generation: 1, epoch: 1 }), now: 1_020 }, + { event: open("api", 2), now: 1_030 }, + { + event: LifecycleEvent.Exited({ + id: "api", + generation: 1, + cause: undefined, + requested: true, + }), + now: 1_040, + }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 2 }), now: 1_050 }, + ]); + + expect(tagsOf(commandsByStep[3])).toEqual(["Stop"]); + expect(tagsOf(commandsByStep[4])).toEqual(["ArmWaiterTimeout"]); + expect(tagsOf(commandsByStep[5])).toEqual(["Launch"]); + expect((at(at(commandsByStep, 5), 0) as Extract).generation).toBe( + 2, + ); + expect(tagsOf(commandsByStep[6])).toEqual(["AdmitConnection"]); + expect(state.services.get("api")?.phase).toEqual({ + _tag: "Running", + generation: 2, + ready: true, + }); + }); + + it("keeps every prerequisite running while a dependent chain runs, and sleeps each one only once its own dependents confirm their stop", () => { + const graph = makeGraph([ + lazy("database"), + lazy("pg-meta", { prerequisites: ["database"] }), + lazy("functions", { prerequisites: ["database"] }), + lazy("studio", { prerequisites: ["pg-meta", "functions"] }), + ]); + let state = initialState(graph, 0); + let commandsByStep: ReadonlyArray>; + + ({ state } = run(state, [{ event: open("studio", 1), now: 0 }])); + expect(state.services.get("database")?.phase).toEqual({ + _tag: "Starting", + generation: 1, + stage: "preparing", + }); + expect(state.services.get("pg-meta")?.phase._tag).toBe("Stopped"); + expect(state.services.get("functions")?.phase._tag).toBe("Stopped"); + + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "database", generation: 1 }), now: 10 }, + ])); + expect(state.services.get("pg-meta")?.phase._tag).toBe("Starting"); + expect(state.services.get("functions")?.phase._tag).toBe("Starting"); + + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "pg-meta", generation: 1 }), now: 20 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "functions", generation: 1 }), now: 30 }, + ])); + expect(state.services.get("studio")?.phase._tag).toBe("Starting"); + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "studio", generation: 1 }), now: 40 }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["AdmitConnection"]); + + // Studio idles and its exit is confirmed: only then do its prerequisites become idle-eligible. + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.ConnectionClosed({ id: "studio" }), now: 50 }, + { event: LifecycleEvent.IdleElapsed({ id: "studio", generation: 1, epoch: 1 }), now: 1_050 }, + ])); + expect(tagsOf(commandsByStep[1])).toEqual(["Stop"]); + expect(state.services.get("pg-meta")?.idleArmedEpoch).toBeUndefined(); + expect(state.services.get("functions")?.idleArmedEpoch).toBeUndefined(); + // The database is a dependent-of-a-dependent: it still sees pg-meta/functions `Running`. + expect(state.services.get("database")?.idleArmedEpoch).toBeUndefined(); + + ({ state, commandsByStep } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "studio", + generation: 1, + cause: undefined, + requested: true, + }), + now: 1_060, + }, + ])); + expect(state.services.get("studio")?.phase).toEqual({ _tag: "Stopped" }); + expect(tagsOf(commandsByStep[0]).sort()).toEqual(["ArmIdleTimer", "ArmIdleTimer"]); + // pg-meta and functions are now merely idle-armed, still `Running`, so the database stays up. + expect(state.services.get("database")?.phase._tag).toBe("Running"); + expect(state.services.get("database")?.idleArmedEpoch).toBeUndefined(); + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.IdleElapsed({ id: "pg-meta", generation: 1, epoch: 1 }), now: 2_060 }, + { + event: LifecycleEvent.IdleElapsed({ id: "functions", generation: 1, epoch: 1 }), + now: 2_060, + }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["Stop"]); + expect(tagsOf(commandsByStep[1])).toEqual(["Stop"]); + + ({ state } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "pg-meta", + generation: 1, + cause: undefined, + requested: true, + }), + now: 2_070, + }, + ])); + // Functions is still running: the database still has a live dependent. + expect(state.services.get("database")?.idleArmedEpoch).toBeUndefined(); + + ({ state, commandsByStep } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "functions", + generation: 1, + cause: undefined, + requested: true, + }), + now: 2_080, + }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["ArmIdleTimer"]); + expect(state.services.get("database")?.idleArmedEpoch).toBe(1); + + ({ state, commandsByStep } = run(state, [ + { + event: LifecycleEvent.IdleElapsed({ id: "database", generation: 1, epoch: 1 }), + now: 3_080, + }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["Stop"]); + + ({ state } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "database", + generation: 1, + cause: undefined, + requested: true, + }), + now: 3_090, + }, + ])); + expect(state.services.get("database")?.phase).toEqual({ _tag: "Stopped" }); + }); + + it("blocks admission across the full prerequisite closure when a transitive prerequisite loses readiness", () => { + const graph = makeGraph([ + lazy("a"), + lazy("b", { prerequisites: ["a"] }), + lazy("c", { prerequisites: ["b"] }), + ]); + let state = initialState(graph, 0); + let commandsByStep: ReadonlyArray>; + + ({ state } = run(state, [{ event: open("c", 1), now: 0 }])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "a", generation: 1 }), now: 10 }, + ])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "b", generation: 1 }), now: 20 }, + ])); + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "c", generation: 1 }), now: 30 }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["AdmitConnection"]); + + ({ state } = run(state, [ + { + event: LifecycleEvent.ReadinessLost({ id: "a", generation: 1, cause: "db down" }), + now: 40, + }, + ])); + // b and c are untouched directly: losing a prerequisite doesn't replace a healthy dependent. + expect(state.services.get("b")?.phase).toEqual({ _tag: "Running", generation: 1, ready: true }); + expect(state.services.get("c")?.phase).toEqual({ _tag: "Running", generation: 1, ready: true }); + + const blocked = run(state, [{ event: open("c", 2), now: 50 }]); + expect(tagsOf(blocked.commandsByStep[0])).toEqual(["ArmWaiterTimeout"]); + expect(blocked.state.services.get("c")?.waiters.size).toBe(1); + + const expired = run(blocked.state, [ + { event: LifecycleEvent.WaiterExpired({ id: "c", waiterId: 2 }), now: 50 + 120_000 + 1 }, + ]); + const failure = commandTagged(at(expired.commandsByStep, 0), "FailConnection"); + expect(failure.message).toContain("prerequisite a"); + }); + + describe("explicit start and restart demand", () => { + it("launches an idle lazy service on its own explicit start, with no connection involved", () => { + const graph = makeGraph([lazy("api")]); + const { commandsByStep } = run(initialState(graph, 0), [ + { event: LifecycleEvent.StartRequested({ id: "api" }), now: 0 }, + ]); + expect(tagsOf(commandsByStep[0])).toEqual(["Launch"]); + }); + + it("restarts after an explicit stop, even though restart alone would otherwise leave intent stopped", () => { + const graph = makeGraph([lazy("api")]); + let state = initialState(graph, 0); + ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "api" }), now: 0 }])); + + const restarted = run(state, [ + { event: LifecycleEvent.RestartRequested({ id: "api" }), now: 1 }, + ]); + expect(tagsOf(restarted.commandsByStep[0])).toEqual(["Launch"]); + expect(restarted.state.services.get("api")?.intent).toBe("lazy"); + }); + + it("propagates an explicit restart's demand to a fully stopped prerequisite", () => { + const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); + const { commandsByStep } = run(initialState(graph, 0), [ + { event: LifecycleEvent.RestartRequested({ id: "b" }), now: 0 }, + ]); + expect(tagsOf(commandsByStep[0])).toEqual(["Launch"]); + expect(commandTagged(at(commandsByStep, 0), "Launch").id).toBe("a"); + }); + }); + + describe("circuit breaker", () => { + it("opens after 3 consecutive failures, fails fast naming the cause, recovers once after the cooldown for several waiters at once, and resets after stable readiness", () => { + const graph = makeGraph([lazy("flaky")]); + let state = initialState(graph, 0); + let commandsByStep: ReadonlyArray>; + + const failAttempt = (waiterId: number, now: number) => { + ({ state, commandsByStep } = run(state, [{ event: open("flaky", waiterId), now }])); + const generation = startingGeneration(state, "flaky"); + ({ state, commandsByStep } = run(state, [ + { + event: LifecycleEvent.LaunchFailed({ + id: "flaky", + generation, + cause: `boom-${generation}`, + }), + now: now + 1, + }, + ])); + return commandsByStep; + }; + + failAttempt(1, 0); + failAttempt(2, 100); + const thirdFailure = failAttempt(3, 200); + expect(at(thirdFailure, 0).some((command) => command._tag === "FailConnection")).toBe(true); + expect(state.services.get("flaky")?.breaker.consecutiveFailures).toBe(3); + expect(state.services.get("flaky")?.breaker.openUntil).toBe(201 + 30_000); + + // Fails fast while the breaker is open, naming the last cause, without a new launch. + ({ state, commandsByStep } = run(state, [{ event: open("flaky", 4), now: 300 }])); + expect(tagsOf(commandsByStep[0])).toEqual(["FailConnection"]); + const fastFail = commandTagged(at(commandsByStep, 0), "FailConnection"); + expect(fastFail.cause).toBe("boom-3"); + expect(state.services.get("flaky")?.phase._tag).toBe("Failed"); + + // The dedicated cooldown event is what reopens admission, not the mere passage of time. + ({ state } = run(state, [ + { + event: LifecycleEvent.CooldownElapsed({ id: "flaky", openUntil: 201 + 30_000 }), + now: 201 + 30_000, + }, + ])); + + // After the cooldown elapses, several waiters share exactly one recovery attempt. + ({ state, commandsByStep } = run(state, [{ event: open("flaky", 5), now: 201 + 30_000 }])); + expect(tagsOf(commandsByStep[0])).toEqual(["ArmWaiterTimeout", "Launch"]); + const recovery = commandTagged(at(commandsByStep, 0), "Launch"); + + ({ state, commandsByStep } = run(state, [ + { event: open("flaky", 6), now: 201 + 30_000 + 1 }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["ArmWaiterTimeout"]); + + ({ state, commandsByStep } = run(state, [ + { + event: LifecycleEvent.LaunchSucceeded({ id: "flaky", generation: recovery.generation }), + now: 201 + 30_000 + 10, + }, + ])); + expect(tagsOf(commandsByStep[0]).toSorted()).toEqual(["AdmitConnection", "AdmitConnection"]); + + // 30s of stable readiness on the same generation resets the breaker lazily, at the next event. + ({ state, commandsByStep } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "flaky", + generation: recovery.generation, + cause: "crash", + requested: false, + }), + now: 201 + 30_000 + 10 + 30_000 + 1, + }, + ])); + expect(state.services.get("flaky")?.breaker.consecutiveFailures).toBe(1); + expect(state.services.get("flaky")?.breaker.openUntil).toBeUndefined(); + }); + + it("doubles the cooldown on a second breaker open and an explicit start resets it", () => { + const graph = makeGraph([lazy("flaky")]); + let state = initialState(graph, 0); + + const fail = (waiterId: number, now: number) => { + ({ state } = run(state, [{ event: open("flaky", waiterId), now }])); + const generation = startingGeneration(state, "flaky"); + ({ state } = run(state, [ + { + event: LifecycleEvent.LaunchFailed({ id: "flaky", generation, cause: "boom" }), + now: now + 1, + }, + ])); + }; + + fail(1, 0); + fail(2, 1); + fail(3, 2); + expect(state.services.get("flaky")?.breaker.openUntil).toBe(3 + 30_000); + + ({ state } = run(state, [ + { + event: LifecycleEvent.CooldownElapsed({ id: "flaky", openUntil: 3 + 30_000 }), + now: 3 + 30_000, + }, + ])); + fail(4, 3 + 30_000); + expect(state.services.get("flaky")?.breaker.consecutiveFailures).toBe(4); + expect(state.services.get("flaky")?.breaker.openUntil).toBe(3 + 30_000 + 1 + 60_000); + + ({ state } = run( + state, + [{ event: LifecycleEvent.StartRequested({ id: "flaky" }) }].map((step) => ({ + ...step, + now: 500_000, + })), + )); + expect(state.services.get("flaky")?.breaker.consecutiveFailures).toBe(0); + expect(state.services.get("flaky")?.breaker.openUntil).toBeUndefined(); + }); + + it("schedules a cooldown timer when the breaker opens and only the dedicated event retries, even with demand retained the whole time", () => { + const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); + let state = initialState(graph, 0); + let commandsByStep: ReadonlyArray>; + + // b's own waiter on a never resolves, so it keeps retaining demand on a across every retry. + ({ state, commandsByStep } = run(state, [{ event: open("b", 1), now: 0 }])); + expect(tagsOf(commandsByStep[0])).toEqual(["ArmWaiterTimeout", "Launch"]); + + const failRetained = (now: number) => { + const generation = startingGeneration(state, "a"); + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.LaunchFailed({ id: "a", generation, cause: "boom" }), now }, + ])); + return commandsByStep; + }; + + // The first two failures retry on their own, because b's waiter still holds demand on a. + failRetained(1); + expect(state.services.get("a")?.phase._tag).toBe("Starting"); + failRetained(2); + const third = failRetained(3); + expect(state.services.get("a")?.phase._tag).toBe("Failed"); + const armed = commandTagged(at(third, 0), "ArmCooldownTimer"); + + // Demand is still retained, but the breaker stays shut until the dedicated event fires. + const stillClosed = run(state, [ + { event: LifecycleEvent.ConnectionClosed({ id: "a" }), now: armed.openUntil - 1 }, + ]); + expect(stillClosed.state.services.get("a")?.phase._tag).toBe("Failed"); + + const elapsed = run(state, [ + { + event: LifecycleEvent.CooldownElapsed({ id: "a", openUntil: armed.openUntil }), + now: armed.openUntil, + }, + ]); + expect(tagsOf(elapsed.commandsByStep[0])).toEqual(["Launch"]); + }); + + it("leaves a stale launch outcome a complete no-op even with demand retained and the cooldown elapsed on the wall clock", () => { + const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); + let state = initialState(graph, 0); + + // b's own waiter on a is never resolved, so demand on a is retained through every failure: + // under the old time-based breaker check, this is exactly the state a stale event could + // have exploited to sneak a new launch past the still-open breaker. + ({ state } = run(state, [{ event: open("b", 1), now: 0 }])); + + const fail = (now: number) => { + const generation = startingGeneration(state, "a"); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchFailed({ id: "a", generation, cause: "boom" }), now }, + ])); + }; + fail(1); + fail(2); + fail(3); + expect(state.services.get("a")?.breaker.openUntil).toBeDefined(); + expect(state.services.get("a")?.phase._tag).toBe("Failed"); + + // Far past the cooldown on the wall clock, but no `CooldownElapsed` has been processed yet. + const stale = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "a", generation: 1 }), now: 10_000_000 }, + ]); + expect(stale.commandsByStep[0]).toEqual([]); + expect(stale.state).toEqual(state); + }); + + it("materializes an already-earned stability reset before a readiness loss discards the evidence", () => { + const graph = makeGraph([lazy("flaky")]); + let state = initialState(graph, 0); + + const fail = (waiterId: number, now: number) => { + ({ state } = run(state, [{ event: open("flaky", waiterId), now }])); + const generation = startingGeneration(state, "flaky"); + ({ state } = run(state, [ + { + event: LifecycleEvent.LaunchFailed({ id: "flaky", generation, cause: "boom" }), + now: now + 1, + }, + ])); + }; + fail(1, 0); + fail(2, 100); + + ({ state } = run(state, [{ event: open("flaky", 3), now: 200 }])); + const generation = startingGeneration(state, "flaky"); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "flaky", generation }), now: 201 }, + ])); + + ({ state } = run(state, [ + { + event: LifecycleEvent.ReadinessLost({ id: "flaky", generation, cause: "blip" }), + now: 201 + 30_000 + 1, + }, + ])); + expect(state.services.get("flaky")?.breaker.consecutiveFailures).toBe(0); + + const result = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "flaky", + generation, + cause: "crash", + requested: false, + }), + now: 201 + 30_000 + 2, + }, + ]); + expect(result.state.services.get("flaky")?.breaker.consecutiveFailures).toBe(1); + expect(result.state.services.get("flaky")?.breaker.openUntil).toBeUndefined(); + }); + + it("materializes an already-earned stability reset before an idle stop starts a new generation", () => { + const graph = makeGraph([lazy("flaky")]); + let state = initialState(graph, 0); + + const fail = (waiterId: number, now: number) => { + ({ state } = run(state, [{ event: open("flaky", waiterId), now }])); + const generation = startingGeneration(state, "flaky"); + ({ state } = run(state, [ + { + event: LifecycleEvent.LaunchFailed({ id: "flaky", generation, cause: "boom" }), + now: now + 1, + }, + ])); + }; + fail(1, 0); + fail(2, 100); + + ({ state } = run(state, [{ event: open("flaky", 3), now: 200 }])); + const generation = startingGeneration(state, "flaky"); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "flaky", generation }), now: 201 }, + { event: LifecycleEvent.ConnectionClosed({ id: "flaky" }), now: 202 }, + ])); + + ({ state } = run(state, [ + { + event: LifecycleEvent.IdleElapsed({ id: "flaky", generation, epoch: 1 }), + now: 202 + 30_000 + 1_000, + }, + ])); + expect(state.services.get("flaky")?.breaker.consecutiveFailures).toBe(0); + + ({ state } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "flaky", + generation, + cause: undefined, + requested: true, + }), + now: 202 + 30_000 + 1_000 + 10, + }, + ])); + expect(state.services.get("flaky")?.phase).toEqual({ _tag: "Stopped" }); + + const nextAttempt = run(state, [{ event: open("flaky", 4), now: 1_000_000 }]); + const nextGeneration = startingGeneration(nextAttempt.state, "flaky"); + const result = run(nextAttempt.state, [ + { + event: LifecycleEvent.LaunchFailed({ + id: "flaky", + generation: nextGeneration, + cause: "boom", + }), + now: 1_000_001, + }, + ]); + expect(result.state.services.get("flaky")?.breaker.consecutiveFailures).toBe(1); + }); + }); + + it("caps waiters at 256, lets an expiry fail only that waiter while the shared launch continues, and frees its slot for cancellation", () => { + const graph = makeGraph([lazy("api")]); + let state = initialState(graph, 0); + let commandsByStep: ReadonlyArray>; + + const opens = Array.from({ length: 256 }, (_, index) => ({ + event: open("api", index + 1), + now: 0, + })); + ({ state } = run(state, opens)); + expect(state.services.get("api")?.waiters.size).toBe(256); + expect(state.services.get("api")?.phase._tag).toBe("Starting"); + + ({ state, commandsByStep } = run(state, [{ event: open("api", 257), now: 1 }])); + expect(tagsOf(commandsByStep[0])).toEqual(["FailConnection"]); + expect(state.services.get("api")?.waiters.size).toBe(256); + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.WaiterExpired({ id: "api", waiterId: 5 }), now: 120_001 }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["FailConnection"]); + const expired = commandTagged(at(commandsByStep, 0), "FailConnection"); + expect(expired.message).toContain("api to finish preparing"); + expect(state.services.get("api")?.waiters.size).toBe(255); + // The shared launch itself is untouched by a waiter's own expiry. + expect(state.services.get("api")?.phase._tag).toBe("Starting"); + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.WaiterCancelled({ id: "api", waiterId: 6 }), now: 2 }, + ])); + expect(commandsByStep[0]).toEqual([]); + expect(state.services.get("api")?.waiters.size).toBe(254); + + ({ state, commandsByStep } = run(state, [{ event: open("api", 300), now: 3 }])); + expect(tagsOf(commandsByStep[0])).toEqual(["ArmWaiterTimeout"]); + expect(state.services.get("api")?.waiters.size).toBe(255); + }); + + it("expires a waiter instead of admitting it when its deadline already passed by the time the launch succeeds", () => { + const graph = makeGraph([lazy("api", { waiterBudgetMillis: 100, idleMillis: undefined })]); + let state = initialState(graph, 0); + ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); + const generation = startingGeneration(state, "api"); + + const result = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation }), now: 500 }, + ]); + expect(tagsOf(result.commandsByStep[0])).toEqual(["FailConnection"]); + expect(result.state.services.get("api")?.leases).toBe(0); + }); + + it("admits a non-ready-requiring acquisition once a live session exists, bypassing target readiness", () => { + const graph = makeGraph([lazy("functions")]); + let state = initialState(graph, 0); + ({ state } = run(state, [{ event: open("functions", 1), now: 0 }])); + const generation = startingGeneration(state, "functions"); + + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "functions", generation }), now: 10 }, + { + event: LifecycleEvent.ReadinessLost({ + id: "functions", + generation, + cause: "still booting", + }), + now: 11, + }, + ])); + expect(state.services.get("functions")?.phase).toEqual({ + _tag: "Running", + generation, + ready: false, + }); + + const readyRequiring = run(state, [{ event: open("functions", 2), now: 20 }]); + expect(tagsOf(readyRequiring.commandsByStep[0])).toEqual(["ArmWaiterTimeout"]); + + const inspector = run(state, [ + { + event: LifecycleEvent.ConnectionOpened({ + id: "functions", + waiterId: 3, + requireReady: false, + }), + now: 20, + }, + ]); + expect(tagsOf(inspector.commandsByStep[0])).toEqual(["AdmitConnection"]); + }); + + it("treats a readiness loss as non-terminal with a reprobe, unlike an exit which counts toward the breaker", () => { + const graph = makeGraph([lazy("api")]); + const { state: ready } = run(initialState(graph, 0), [ + { event: open("api", 1), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, + ]); + + const { state: afterReadinessLoss, commandsByStep: lostCommands } = run(ready, [ + { + event: LifecycleEvent.ReadinessLost({ id: "api", generation: 1, cause: "probe failed" }), + now: 20, + }, + ]); + expect(tagsOf(lostCommands[0])).toEqual(["Reprobe"]); + expect(afterReadinessLoss.services.get("api")?.phase).toEqual({ + _tag: "Running", + generation: 1, + ready: false, + }); + expect(afterReadinessLoss.services.get("api")?.breaker.consecutiveFailures).toBe(0); + + const { state: recovered } = run(afterReadinessLoss, [ + { event: LifecycleEvent.ReadinessRecovered({ id: "api", generation: 1 }), now: 30 }, + ]); + expect(recovered.services.get("api")?.phase).toEqual({ + _tag: "Running", + generation: 1, + ready: true, + }); + expect(recovered.services.get("api")?.breaker.consecutiveFailures).toBe(0); + + // Closing the lease first removes all demand, so the crash below settles into `Failed` + // instead of an immediate crash re-wake, isolating the phase/breaker assertion that follows. + const { state: idle } = run(ready, [ + { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 15 }, + ]); + const { state: afterExit, commandsByStep: exitCommands } = run(idle, [ + { + event: LifecycleEvent.Exited({ + id: "api", + generation: 1, + cause: "crash", + requested: false, + }), + now: 20, + }, + ]); + expect(afterExit.services.get("api")?.phase).toMatchObject({ _tag: "Failed", cause: "crash" }); + expect(afterExit.services.get("api")?.breaker.consecutiveFailures).toBe(1); + expect(exitCommands[0]).toEqual([]); + }); + + it("rejects stopping or restarting a prerequisite while a dependent is active", () => { + const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); + let state = initialState(graph, 0); + ({ state } = run(state, [{ event: open("b", 1), now: 0 }])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "a", generation: 1 }), now: 10 }, + ])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "b", generation: 1 }), now: 20 }, + ])); + expect(state.services.get("b")?.leases).toBe(1); + + const stopResult = run(state, [{ event: LifecycleEvent.StopRequested({ id: "a" }), now: 30 }]); + expect(tagsOf(stopResult.commandsByStep[0])).toEqual(["RequestRejected"]); + expect(stopResult.state.services.get("a")?.phase._tag).toBe("Running"); + + const restartResult = run(state, [ + { event: LifecycleEvent.RestartRequested({ id: "a" }), now: 30 }, + ]); + expect(tagsOf(restartResult.commandsByStep[0])).toEqual(["RequestRejected"]); + expect(restartResult.state.services.get("a")?.phase).toEqual({ + _tag: "Running", + generation: 1, + ready: true, + }); + }); + + describe("storage reservation", () => { + it("admits storage operations only once a service is stopped with intent stopped", () => { + const graph = makeGraph([lazy("database")]); + let state = initialState(graph, 0); + expect(canRunStorage(state, "database")).toBe(false); + + ({ state } = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "database" }), now: 0 }, + ])); + expect(canRunStorage(state, "database")).toBe(true); + + ({ state } = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "database" }), now: 1 }, + ])); + ({ state } = run(state, [{ event: open("database", 1), now: 2 }])); + expect(canRunStorage(state, "database")).toBe(false); + }); + + it("rejects a storage reservation unless the service is already stopped", () => { + const graph = makeGraph([lazy("database")]); + const { commandsByStep } = run(initialState(graph, 0), [ + { event: LifecycleEvent.StorageReserved({ id: "database" }), now: 0 }, + ]); + expect(tagsOf(commandsByStep[0])).toEqual(["RequestRejected"]); + }); + + it("blocks a launch while storage is reserved and relaunches once it's released", () => { + const graph = makeGraph([lazy("database")]); + let state = initialState(graph, 0); + ({ state } = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "database" }), now: 0 }, + ])); + + const reserved = run(state, [ + { event: LifecycleEvent.StorageReserved({ id: "database" }), now: 1 }, + ]); + expect(reserved.commandsByStep[0]).toEqual([]); + expect(reserved.state.services.get("database")?.storageReserved).toBe(true); + state = reserved.state; + + // An explicit start during the restore can't emit a launch while storage is reserved. + const duringRestore = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "database" }), now: 2 }, + ]); + expect(duringRestore.commandsByStep[0]).toEqual([]); + state = duringRestore.state; + + const released = run(state, [ + { event: LifecycleEvent.StorageReleased({ id: "database" }), now: 3 }, + ]); + expect(tagsOf(released.commandsByStep[0])).toEqual(["Launch"]); + }); + }); + + it("rejects new admission for an explicitly stopped service without creating a waiter", () => { + const graph = makeGraph([lazy("api")]); + let state = initialState(graph, 0); + ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "api" }), now: 0 }])); + + const result = run(state, [{ event: open("api", 1), now: 1 }]); + expect(tagsOf(result.commandsByStep[0])).toEqual(["FailConnection"]); + expect(result.state.services.get("api")?.waiters.size).toBe(0); + }); + + it("starts an eager service's prerequisite on its own and cascades once that prerequisite is ready", () => { + const graph = makeGraph([ + lazy("database", { idleMillis: undefined }), + { id: "kong", activation: "eager", prerequisites: ["database"] }, + ]); + let state = initialState(graph, 0); + let commandsByStep: ReadonlyArray>; + + // Kong is eager from composition, so the owner's first settle pass (triggered by any event, + // here the composition's own start signal) already starts its prerequisite without a waiter. + ({ state } = run(state, [{ event: LifecycleEvent.StartRequested({ id: "kong" }), now: 0 }])); + expect(state.services.get("database")?.phase._tag).toBe("Starting"); + expect(state.services.get("kong")?.phase._tag).toBe("Stopped"); + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "database", generation: 1 }), now: 10 }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["Launch"]); + expect(state.services.get("kong")?.phase._tag).toBe("Starting"); + }); + + describe("graph updates", () => { + it("preserves a continuing service's state and never reuses a generation after a remove and re-add", () => { + let state = initialState(makeGraph([lazy("api")]), 0); + ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, + ])); + ({ state } = run(state, [ + { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 20 }, + ])); + ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "api" }), now: 30 }])); + ({ state } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "api", + generation: 1, + cause: undefined, + requested: true, + }), + now: 31, + }, + ])); + expect(state.services.get("api")?.phase).toEqual({ _tag: "Stopped" }); + + ({ state } = run(state, [ + { event: LifecycleEvent.GraphUpdated({ graph: makeGraph([]) }), now: 40 }, + ])); + expect(state.services.has("api")).toBe(false); + + ({ state } = run(state, [ + { event: LifecycleEvent.GraphUpdated({ graph: makeGraph([lazy("api")]) }), now: 50 }, + ])); + expect(state.services.get("api")?.phase).toEqual({ _tag: "Stopped" }); + + const relaunch = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "api" }), now: 60 }, + ]); + expect(commandTagged(at(relaunch.commandsByStep, 0), "Launch").generation).toBe(2); + }); + + it("rejects removing a service that's starting with a pending waiter, atomically", () => { + let state = initialState(makeGraph([lazy("a")]), 0); + ({ state } = run(state, [{ event: open("a", 1), now: 0 }])); + expect(state.services.get("a")?.phase._tag).toBe("Starting"); + + const rejected = run(state, [ + { event: LifecycleEvent.GraphUpdated({ graph: makeGraph([]) }), now: 10 }, + ]); + expect(tagsOf(rejected.commandsByStep[0])).toEqual(["RequestRejected"]); + expect(rejected.state).toEqual(state); + }); + + it("rejects removing a service with a storage reservation, so a graph update can't launch mid-restore", () => { + let state = initialState(makeGraph([lazy("db")]), 0); + ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "db" }), now: 0 }])); + ({ state } = run(state, [{ event: LifecycleEvent.StorageReserved({ id: "db" }), now: 1 }])); + expect(state.services.get("db")?.storageReserved).toBe(true); + + const rejected = run(state, [ + { event: LifecycleEvent.GraphUpdated({ graph: makeGraph([]) }), now: 2 }, + ]); + expect(tagsOf(rejected.commandsByStep[0])).toEqual(["RequestRejected"]); + expect(rejected.state).toEqual(state); + }); + + it("rejects changing an active continuing service's prerequisites, leaving its live generation untouched", () => { + const graphV1 = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); + let state = initialState(graphV1, 0); + ({ state } = run(state, [{ event: open("b", 1), now: 0 }])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "a", generation: 1 }), now: 10 }, + ])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "b", generation: 1 }), now: 20 }, + ])); + expect(state.services.get("b")?.leases).toBe(1); + + // b keeps its id but drops a from its prerequisites, while b is still actively running on a. + const graphV2 = makeGraph([lazy("a"), lazy("b", { prerequisites: [] })]); + const rejected = run(state, [ + { event: LifecycleEvent.GraphUpdated({ graph: graphV2 }), now: 30 }, + ]); + expect(tagsOf(rejected.commandsByStep[0])).toEqual(["RequestRejected"]); + expect(rejected.state).toEqual(state); + + // The update never took effect: a is still b's prerequisite and still can't be stopped. + const stopResult = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "a" }), now: 40 }, + ]); + expect(tagsOf(stopResult.commandsByStep[0])).toEqual(["RequestRejected"]); + }); + }); + + describe("stage reporting", () => { + it("updates the starting stage so a waiter's budget error names it", () => { + let state = initialState(makeGraph([lazy("api", { waiterBudgetMillis: 50 })]), 0); + ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); + const generation = startingGeneration(state, "api"); + + ({ state } = run(state, [ + { + event: LifecycleEvent.StageChanged({ id: "api", generation, stage: "artifacts" }), + now: 1, + }, + ])); + expect(state.services.get("api")?.phase).toEqual({ + _tag: "Starting", + generation, + stage: "artifacts", + }); + + const expired = run(state, [ + { event: LifecycleEvent.WaiterExpired({ id: "api", waiterId: 1 }), now: 100 }, + ]); + const failure = commandTagged(at(expired.commandsByStep, 0), "FailConnection"); + expect(failure.message).toContain("api to finish artifacts"); + }); + + it("ignores a stage change tagged with a stale generation", () => { + let state = initialState(makeGraph([lazy("api")]), 0); + ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); + const result = run(state, [ + { + event: LifecycleEvent.StageChanged({ id: "api", generation: 99, stage: "artifacts" }), + now: 1, + }, + ]); + expect(result.state).toEqual(state); + }); + }); + + describe("the dependent guard", () => { + it("protects stranded downstream demand even when the immediate dependent is itself inactive", () => { + const graph = makeGraph([ + lazy("a"), + lazy("b", { prerequisites: ["a"] }), + lazy("c", { prerequisites: ["b"] }), + ]); + let state = initialState(graph, 0); + ({ state } = run(state, [{ event: open("c", 1), now: 0 }])); + expect(state.services.get("a")?.phase._tag).toBe("Starting"); + expect(state.services.get("b")?.phase._tag).toBe("Stopped"); + expect(state.services.get("c")?.waiters.size).toBe(1); + + const stopResult = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "a" }), now: 10 }, + ]); + expect(tagsOf(stopResult.commandsByStep[0])).toEqual(["RequestRejected"]); + expect(stopResult.state).toEqual(state); + }); + + it("still protects a dependent's demand once the prerequisite's own breaker has opened", () => { + const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); + let state = initialState(graph, 0); + ({ state } = run(state, [{ event: open("b", 1), now: 0 }])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "a", generation: 1 }), now: 10 }, + ])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "b", generation: 1 }), now: 20 }, + ])); + expect(state.services.get("b")?.leases).toBe(1); + + // b's lease retains demand on a, so each crash immediately relaunches the next generation, + // until the third failure opens the breaker and a finally rests in `Failed`. + const crash = (now: number) => { + const generation = currentGeneration(state, "a"); + ({ state } = run(state, [ + { + event: LifecycleEvent.Exited({ id: "a", generation, cause: "crash", requested: false }), + now, + }, + ])); + }; + crash(30); + crash(31); + crash(32); + expect(state.services.get("a")?.breaker.openUntil).toBeDefined(); + expect(state.services.get("a")?.phase._tag).toBe("Failed"); + + const stopResult = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "a" }), now: 40 }, + ]); + expect(tagsOf(stopResult.commandsByStep[0])).toEqual(["RequestRejected"]); + expect(stopResult.state.services.get("a")?.phase._tag).toBe("Failed"); + + const restartResult = run(state, [ + { event: LifecycleEvent.RestartRequested({ id: "a" }), now: 40 }, + ]); + expect(tagsOf(restartResult.commandsByStep[0])).toEqual(["RequestRejected"]); + }); + + it("blocks an explicit stop or restart of a prerequisite while a sleeping dependent is still wake-armed", () => { + const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); + let state = initialState(graph, 0); + ({ state } = run(state, [{ event: open("b", 1), now: 0 }])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "a", generation: 1 }), now: 10 }, + ])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "b", generation: 1 }), now: 20 }, + ])); + ({ state } = run(state, [{ event: LifecycleEvent.ConnectionClosed({ id: "b" }), now: 30 }])); + expect(state.services.get("b")?.idleArmedEpoch).toBe(1); + + ({ state } = run(state, [ + { event: LifecycleEvent.IdleElapsed({ id: "b", generation: 1, epoch: 1 }), now: 1_030 }, + ])); + ({ state } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "b", + generation: 1, + cause: undefined, + requested: true, + }), + now: 1_040, + }, + ])); + expect(state.services.get("b")?.phase).toEqual({ _tag: "Stopped" }); + expect(state.services.get("b")?.intent).toBe("lazy"); + + // b is asleep, not explicitly stopped, so it can still wake on its own: a can't be torn down. + const stopResult = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "a" }), now: 1_050 }, + ]); + expect(tagsOf(stopResult.commandsByStep[0])).toEqual(["RequestRejected"]); + expect(stopResult.state.services.get("a")?.phase._tag).toBe("Running"); + + const restartResult = run(state, [ + { event: LifecycleEvent.RestartRequested({ id: "a" }), now: 1_050 }, + ]); + expect(tagsOf(restartResult.commandsByStep[0])).toEqual(["RequestRejected"]); + + // Once b is explicitly stopped too, a's stop is no longer blocked. + ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "b" }), now: 1_060 }])); + const stopAfter = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "a" }), now: 1_070 }, + ]); + expect(tagsOf(stopAfter.commandsByStep[0])).not.toContain("RequestRejected"); + }); + }); + + it("re-arms a stopped prerequisite when its dependent is explicitly restarted", () => { + const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); + let state = initialState(graph, 0); + ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "b" }), now: 0 }])); + ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "a" }), now: 1 }])); + expect(state.services.get("a")?.intent).toBe("stopped"); + + const restarted = run(state, [{ event: LifecycleEvent.RestartRequested({ id: "b" }), now: 2 }]); + expect(restarted.state.services.get("a")?.intent).toBe("lazy"); + expect(tagsOf(restarted.commandsByStep[0])).toEqual(["Launch"]); + expect(commandTagged(at(restarted.commandsByStep, 0), "Launch").id).toBe("a"); + }); + + it("keeps an explicit start's demand through a committed stop so the next generation launches once the exit is confirmed", () => { + let state = initialState(makeGraph([lazy("api")]), 0); + ({ state } = run(state, [{ event: LifecycleEvent.StartRequested({ id: "api" }), now: 0 }])); + expect(state.services.get("api")?.phase._tag).toBe("Starting"); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, + ])); + expect(state.services.get("api")?.leases).toBe(0); + + ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "api" }), now: 20 }])); + expect(state.services.get("api")?.phase._tag).toBe("Stopping"); + + ({ state } = run(state, [{ event: LifecycleEvent.StartRequested({ id: "api" }), now: 21 }])); + expect(state.services.get("api")?.relaunchForced).toBe(true); + + const result = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "api", + generation: 1, + cause: undefined, + requested: true, + }), + now: 30, + }, + ]); + expect(tagsOf(result.commandsByStep[0])).toEqual(["Launch"]); + expect(commandTagged(at(result.commandsByStep, 0), "Launch").generation).toBe(2); + }); + + it("admits a cold inspector acquisition before the first readiness check while ordinary traffic still waits", () => { + let state = initialState(makeGraph([lazy("functions")]), 0); + ({ state } = run(state, [{ event: open("functions", 1), now: 0 }])); + const generation = startingGeneration(state, "functions"); + expect(state.services.get("functions")?.waiters.size).toBe(1); + + const inspectorQueued = run(state, [ + { + event: LifecycleEvent.ConnectionOpened({ + id: "functions", + waiterId: 2, + requireReady: false, + }), + now: 1, + }, + ]); + expect(tagsOf(inspectorQueued.commandsByStep[0])).toEqual(["ArmWaiterTimeout"]); + state = inspectorQueued.state; + + const result = run(state, [ + { event: LifecycleEvent.SessionAvailable({ id: "functions", generation }), now: 2 }, + ]); + expect(tagsOf(result.commandsByStep[0])).toEqual(["AdmitConnection"]); + expect(commandTagged(at(result.commandsByStep, 0), "AdmitConnection").waiterId).toBe(2); + expect(result.state.services.get("functions")?.waiters.size).toBe(1); + + const ready = run(result.state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "functions", generation }), now: 3 }, + ]); + expect(tagsOf(ready.commandsByStep[0])).toEqual(["AdmitConnection"]); + expect(commandTagged(at(ready.commandsByStep, 0), "AdmitConnection").waiterId).toBe(1); + }); + + it("re-arms the idle timer once demand returns and readiness later recovers, instead of staying armed forever", () => { + let state = initialState(makeGraph([lazy("api")]), 0); + ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, + ])); + ({ state } = run(state, [{ event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 20 }])); + expect(state.services.get("api")?.idleArmedEpoch).toBe(1); + + ({ state } = run(state, [ + { event: LifecycleEvent.ReadinessLost({ id: "api", generation: 1, cause: "blip" }), now: 30 }, + ])); + expect(state.services.get("api")?.idleArmedEpoch).toBe(1); + + ({ state } = run(state, [{ event: open("api", 2), now: 40 }])); + expect(state.services.get("api")?.idleArmedEpoch).toBeUndefined(); + + const staleElapse = run(state, [ + { event: LifecycleEvent.IdleElapsed({ id: "api", generation: 1, epoch: 1 }), now: 50 }, + ]); + expect(staleElapse.commandsByStep[0]).toEqual([]); + + ({ state } = run(state, [ + { event: LifecycleEvent.WaiterExpired({ id: "api", waiterId: 2 }), now: 120_040 }, + ])); + ({ state } = run(state, [ + { event: LifecycleEvent.ReadinessRecovered({ id: "api", generation: 1 }), now: 120_041 }, + ])); + expect(state.services.get("api")?.idleArmedEpoch).toBe(2); + }); +}); From e7becbf89e8169d31c80fe53b13ce6f02548c7fe Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sun, 4 Oct 2026 16:01:34 +0200 Subject: [PATCH 027/142] fix(stack): bind native Functions, Storage, PostgREST and Analytics to loopback --- packages/stack/src/services/Analytics.ts | 1 + packages/stack/src/services/Functions.ts | 4 +++- packages/stack/src/services/Rest.ts | 3 ++- packages/stack/src/services/Storage.ts | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/stack/src/services/Analytics.ts b/packages/stack/src/services/Analytics.ts index a8d25fd4b2..a5b7a209c6 100644 --- a/packages/stack/src/services/Analytics.ts +++ b/packages/stack/src/services/Analytics.ts @@ -33,6 +33,7 @@ export const makeSpec = (): ProcessRecipeSpec => ({ const db = yield* databaseConnection(databaseUrl); return { DATABASE_URL: databaseUrl, + PHX_HTTP_IP: container ? "0.0.0.0" : "127.0.0.1", ...(http === undefined ? {} : { PORT: String(http.port), PHX_HTTP_PORT: String(http.port) }), diff --git a/packages/stack/src/services/Functions.ts b/packages/stack/src/services/Functions.ts index c03bdec3b2..3707a4d9f6 100644 --- a/packages/stack/src/services/Functions.ts +++ b/packages/stack/src/services/Functions.ts @@ -196,7 +196,9 @@ export const makeSpec = ( return [ "start", `--main-service=${root}`, - ...(http === undefined ? [] : [`--port=${http.port}`]), + ...(http === undefined + ? [] + : [`--port=${http.port}`, `--ip=${context.container ? "0.0.0.0" : "127.0.0.1"}`]), ...(creation.config.policy === undefined ? [] : [`--policy=${creation.config.policy}`]), ...(creation.config.inspector === true && inspector !== undefined ? [`--inspect=${context.container ? "0.0.0.0" : "127.0.0.1"}:${inspector.port}`] diff --git a/packages/stack/src/services/Rest.ts b/packages/stack/src/services/Rest.ts index bbd80ef641..31a8f00aba 100644 --- a/packages/stack/src/services/Rest.ts +++ b/packages/stack/src/services/Rest.ts @@ -27,7 +27,7 @@ export const makeSpec = (): ProcessRecipeSpec => ({ executable: "bin/postgrest", ports: { http: 3000 }, healthPath: "/", - env: (creation, endpoints) => + env: (creation, endpoints, container) => Effect.gen(function* () { const databaseUrl = yield* requiredInput("rest", "databaseUrl", creation.config.databaseUrl); const http = endpoints.get("http"); @@ -35,6 +35,7 @@ export const makeSpec = (): ProcessRecipeSpec => ({ return { DATABASE_URL: databaseUrl, PGRST_DB_URI: databaseUrl, + PGRST_SERVER_HOST: container ? "0.0.0.0" : "127.0.0.1", ...(http === undefined ? {} : { PGRST_SERVER_PORT: String(http.port) }), PGRST_DB_SCHEMAS: creation.config.schemas ?? "public,graphql_public", ...(creation.config.extraSearchPath === undefined diff --git a/packages/stack/src/services/Storage.ts b/packages/stack/src/services/Storage.ts index c91e78d3b1..7e97c0d68d 100644 --- a/packages/stack/src/services/Storage.ts +++ b/packages/stack/src/services/Storage.ts @@ -63,6 +63,7 @@ export const makeSpec = (): ProcessRecipeSpec => ({ const filePath = container ? "/mnt" : creation.config.filePath; return { DATABASE_URL: databaseUrl, + SERVER_HOST: container ? "0.0.0.0" : "127.0.0.1", ...(http === undefined ? {} : { STORAGE_PORT: String(http.port), PORT: String(http.port) }), ANON_KEY: creation.config.anonKey ?? anon, SERVICE_KEY: creation.config.serviceRoleKey ?? service, From 3787aa41cfefcc03f833047086261a745f071226 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Sun, 4 Oct 2026 23:37:08 +0200 Subject: [PATCH 028/142] fix(stack): bind native Realtime, Pooler, and Analytics gRPC to loopback Set the opt-in PHX_HTTP_IP, SUPAVISOR_BIND_IP, and LOGFLARE_GRPC_IP env vars to 127.0.0.1 in native mode so these listeners stop binding the wildcard address once the rebuilt artifacts support them; published r0 binaries ignore the vars, so native mode is unaffected until then. --- packages/stack/src/services/Analytics.ts | 2 +- packages/stack/src/services/Pooler.ts | 1 + packages/stack/src/services/Realtime.ts | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/stack/src/services/Analytics.ts b/packages/stack/src/services/Analytics.ts index a5b7a209c6..0c3e334d47 100644 --- a/packages/stack/src/services/Analytics.ts +++ b/packages/stack/src/services/Analytics.ts @@ -45,7 +45,7 @@ export const makeSpec = (): ProcessRecipeSpec => ({ DB_PASSWORD: db.password ?? "postgres", LOGFLARE_SUPABASE_MODE: "true", LOGFLARE_SINGLE_TENANT: "true", - ...(container ? {} : { LOGFLARE_GRPC_PORT: "0" }), + ...(container ? {} : { LOGFLARE_GRPC_PORT: "0", LOGFLARE_GRPC_IP: "127.0.0.1" }), ...(creation.config.apiKey === undefined ? {} : { LOGFLARE_PRIVATE_ACCESS_TOKEN: creation.config.apiKey }), diff --git a/packages/stack/src/services/Pooler.ts b/packages/stack/src/services/Pooler.ts index 7e58fb7b11..b49f41ff34 100644 --- a/packages/stack/src/services/Pooler.ts +++ b/packages/stack/src/services/Pooler.ts @@ -55,6 +55,7 @@ const environment: ProcessRecipeSpec["env"] = (creation, endpoints, co PROXY_PORT: "0", SESSION_PROXY_PORTS: "0", TRANSACTION_PROXY_PORTS: "0", + SUPAVISOR_BIND_IP: "127.0.0.1", }), ...(sql === undefined ? {} diff --git a/packages/stack/src/services/Realtime.ts b/packages/stack/src/services/Realtime.ts index 87a7a46f35..231b8e3d13 100644 --- a/packages/stack/src/services/Realtime.ts +++ b/packages/stack/src/services/Realtime.ts @@ -52,6 +52,7 @@ export const makeSpec = (): ProcessRecipeSpec => ({ return { DATABASE_URL: databaseUrl, ...(http === undefined ? {} : { PORT: String(http.port) }), + ...(container ? {} : { PHX_HTTP_IP: "127.0.0.1" }), DB_URL: databaseUrl, DB_HOST: db.host, DB_PORT: db.port, From 7df785f0de99077202284047d0ebfbf0a62f375c Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 00:28:03 +0200 Subject: [PATCH 029/142] test(stack): show the response body when a whole-stack status assertion fails --- packages/stack/tests/whole-stack/scenarios.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/stack/tests/whole-stack/scenarios.ts b/packages/stack/tests/whole-stack/scenarios.ts index 75c0259744..ce10ddcf00 100644 --- a/packages/stack/tests/whole-stack/scenarios.ts +++ b/packages/stack/tests/whole-stack/scenarios.ts @@ -191,7 +191,7 @@ const exerciseStack = Effect.fn("WholeStack.exerciseStack")( const email = `whole-${phase}-${fixture.stack.id}@example.test`; const password = "whole-stack-password"; const signup = yield* jsonRequest("POST", `${authUrl}/signup`, { email, password }); - expect(signup.status).toBe(200); + expect(signup.status, signup.body).toBe(200); const signupBody = yield* Schema.decodeUnknownEffect( Schema.Struct({ access_token: Schema.String, @@ -217,12 +217,12 @@ const exerciseStack = Effect.fn("WholeStack.exerciseStack")( { id: rowId, owner_id: signupBody.user.id, value: phase }, { ...headers, prefer: "return=representation" }, ); - expect(insert.status).toBe(201); + expect(insert.status, insert.body).toBe(201); const read = yield* requestWithHeaders( `${restUrl}/whole_stack_items?id=eq.${rowId}`, headers, ); - expect(read.status).toBe(200); + expect(read.status, read.body).toBe(200); const rows = yield* Schema.decodeEffect( Schema.fromJsonString( Schema.Array(Schema.Struct({ id: Schema.String, value: Schema.String })), @@ -234,7 +234,7 @@ const exerciseStack = Effect.fn("WholeStack.exerciseStack")( email: secondEmail, password, }); - expect(secondSignup.status).toBe(200); + expect(secondSignup.status, secondSignup.body).toBe(200); const secondSignupBody = yield* Schema.decodeUnknownEffect( Schema.Struct({ access_token: Schema.String }), )(yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))(secondSignup.body)); @@ -242,7 +242,7 @@ const exerciseStack = Effect.fn("WholeStack.exerciseStack")( authorization: `Bearer ${secondSignupBody.access_token}`, apikey: secondSignupBody.access_token, }); - expect(otherRead.status).toBe(200); + expect(otherRead.status, otherRead.body).toBe(200); expect( yield* Schema.decodeEffect( Schema.fromJsonString(Schema.Array(Schema.Struct({ id: Schema.String }))), @@ -263,7 +263,7 @@ const exerciseStack = Effect.fn("WholeStack.exerciseStack")( return yield* Effect.die("Mail or Realtime URL missing"); const watcher = yield* watchRecoveryMail(mailUrl, email); const recovery = yield* jsonRequest("POST", `${authUrl}/recover`, { email }); - expect(recovery.status).toBe(200); + expect(recovery.status, recovery.body).toBe(200); const mail = yield* watcher.awaitFullMail; expect(mail.To.map((recipient) => recipient.Address)).toContain(email); const authOrigin = new URL(authUrl).origin; @@ -288,7 +288,7 @@ const exerciseStack = Effect.fn("WholeStack.exerciseStack")( { value: `${phase}-realtime` }, { ...headers, prefer: "return=representation" }, ); - expect(update.status).toBe(200); + expect(update.status, update.body).toBe(200); const change = yield* realtime.nextChange; const event = yield* Schema.decodeUnknownEffect( Schema.Struct({ record: Schema.Struct({ id: Schema.String, value: Schema.String }) }), @@ -468,7 +468,7 @@ export const parallel = (runtime: Runtime) => authorization: `Bearer ${leftLedger.accessToken}`, apikey: leftLedger.accessToken, }); - expect(crossStack.status).toBe(401); + expect(crossStack.status, crossStack.body).toBe(401); yield* Effect.scoped( Effect.gen(function* () { const rightRealtime = yield* subscribeRealtime( @@ -520,7 +520,7 @@ export const parallel = (runtime: Runtime) => prefer: "return=representation", }, ); - expect(update.status).toBe(200); + expect(update.status, update.body).toBe(200); const change = yield* rightRealtime.nextChange; const event = yield* Schema.decodeUnknownEffect( Schema.Struct({ record: Schema.Struct({ id: Schema.String, value: Schema.String }) }), From 00bce64b1f53565a4cbb933cb70a728fdf7896d3 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 00:45:23 +0200 Subject: [PATCH 030/142] fix(stack): pin the Realtime, Pooler and Analytics r1 artifacts that accept a loopback bind address --- packages/stack/src/Artifacts.ts | 48 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index d57d02560e..c764bdfa5b 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -208,22 +208,22 @@ const definitions: Readonly> = { "realtime", { upstreamVersion: "v2.140.7", - revision: 0, + revision: 1, image: - "ghcr.io/supabase/cli/realtime:v2.140.7-r0@sha256:ad135917b87f7e0056d87d0ddc97e5a39a7711852f4ea6ec4b3f260a47d79927", + "ghcr.io/supabase/cli/realtime:v2.140.7-r1@sha256:898beb192146adbe2689e62154ec5c8e2a22c55e504592cf42f12698fb6b34ab", upstreamImage: "supabase/realtime:v2.140.7", natives: { "darwin-arm64": { - archive: "5f3ce154d15384b358a872db65b4e25475238816838e0fa767a97336ca6364a8", - manifest: "63758ba9c6b4899929f4d5ba33961f1e1746460d916bf7e27f3202bdd80fb1a8", + archive: "562e5f101fab5d725aa8446ca2ba60ea602e4e8caabea184ff8c168e95a5166c", + manifest: "0105c1910006e336e88a6cb6441bbfcf4b56f32ba239027fead8ffa9ce98958e", }, "linux-amd64": { - archive: "f6746785a4940a40b72ad94514e0836a89a760c819e39e5bbebce567ff5d3a88", - manifest: "8af0cc1aa5770e54ed04b2d7c510d9f6ae59928d3c39a77e8e41b11aac89301e", + archive: "9b8d12d5ef5e4a79f755ab312847cd609561b9752317808804456ff648b7d7c0", + manifest: "f1a7d9c7abe5a25728a6aff999ccd2dfec06d2da3b62c055bade24db8be560d4", }, "linux-arm64": { - archive: "3b0764ded78e46c242979a0598743a92aa41649858742b80bfb572998a55b9c4", - manifest: "f7d78dd862a46fab8f5f409ee00f5d8510e986137cd47705ace0bf39094fae9e", + archive: "cbafba128fbc3733c66d6b9d6567a7e3bc2bc1e6674160000d5c187345ccd63c", + manifest: "db3f535d8087a68a23dde84612fdd5c1b881f1e6c8310f697f6cba9026c879ea", }, }, }, @@ -385,22 +385,22 @@ const definitions: Readonly> = { "analytics", { upstreamVersion: "v1.50.15", - revision: 0, + revision: 1, image: - "ghcr.io/supabase/cli/analytics:v1.50.15-r0@sha256:cea1595bafa6ab32df4854d407261ddef6e35394e300ae29770ba48c49c8dc7e", + "ghcr.io/supabase/cli/analytics:v1.50.15-r1@sha256:2b03ec3120effe5d33c1e2ec4570a932fb60ce6a6851b4920a22388b4e8356c1", upstreamImage: "supabase/logflare:1.50.15", natives: { "darwin-arm64": { - archive: "336c78c501aff270b0fcd2a42229b76d9f2feafa646f49a59c3de29358e3de17", - manifest: "d0e1a0c3fa0a4ebfcebbe490f290699424e3b8c92a542f50111ff88d6e47b8b5", + archive: "abfc9ba64e354129e8f8b0f38b43747a5dab48785fd3593d11e52cd6e0ff37bf", + manifest: "60647643f5521089978dc20c6966b86ed939b9751b187cbb70e0de3fbd2479ac", }, "linux-amd64": { - archive: "a699ce1522dac43989987a9d152648ed0506fb887ffedff49c148da02cbbc7e7", - manifest: "ba8c55eeb06be91ad582156854babdc6a06d7a26fa6956bc1271eee4cae9d536", + archive: "7652456c1a1d73ac6fe10853bd8a38e79139b43fe796caec5725a3d0f225a3d8", + manifest: "6c465af69e26ce8a9c15988d5dcbbae069e9a92b697962bfa86c4f3e601b8409", }, "linux-arm64": { - archive: "bb977b9cd0623e1b1ece9875377fdf29577f96991d2e5c72a38a1b664c6a7538", - manifest: "1739bbd4afb838e62b6191c4de8aa39685df9f5d83e1b7cc303bd096237ba08c", + archive: "a75625d7c903d70e749b5ee776e2c5c98d51ee60ff92b28bb6684121fe0f3888", + manifest: "ca2569304cf24e6abd4242b55dd0d20338f40532fbe6377a9accdc5d47ea47ab", }, }, }, @@ -437,22 +437,22 @@ const definitions: Readonly> = { "pooler", { upstreamVersion: "v2.9.13", - revision: 0, + revision: 1, image: - "ghcr.io/supabase/cli/pooler:v2.9.13-r0@sha256:3a32b56d03675ed24e84408afcbb12fa52b7ec6e7741f913770fea5b66c2ddd3", + "ghcr.io/supabase/cli/pooler:v2.9.13-r1@sha256:f02ccc6e18ea77789978e248ac302a4e0955d03fdeeea0d63ac0b7a315595c60", upstreamImage: "supabase/supavisor:2.9.13", natives: { "darwin-arm64": { - archive: "c05285be2a945a29d5be491661926f8c88d976540d49ddbcab10aa7276b29934", - manifest: "0fafa8dcf601ff3cea7326e82f7a5191ffe03e3e4edd0d2b155e80e8c9f53d78", + archive: "aa53594e91144a6d33267b802152f0f42d35ca85e6efd13b9cd06f53e327fb5f", + manifest: "1c42764f1f706cb177ee4f066c6ebcb75e15a2399e8f552488a426718d9e2edd", }, "linux-amd64": { - archive: "d94e36f44267b4159fa55e1aea39320a68789b6af07f9cc87c27327ed6a00602", - manifest: "4f181d1d25da91f72ca37c22cdc46278a424a4edef6bdd2ea2c30f463c629a5a", + archive: "ff1543cebb9331646294475c609c3e231618e46f1dca53108d35f3172c1fa705", + manifest: "1ef9f8041632ca0c96045f64092c880a3ad7a6af4febe7d0b43322b330fc19f8", }, "linux-arm64": { - archive: "124f3e6c95e6ba985239e013cab76c5be28fbd92cb4986b687dd5e827f0525ca", - manifest: "c6cdc298bc3a00d082a1c9c34f4131f13437df51fb0bf9a0ce857b0a2fd66460", + archive: "ed08b8856a0516026321a47055fca5c30cea82f93dc3da8bb2ab81951342a20c", + manifest: "12e867a370fac6aa68f1e657840d30c9d4e5cae56f38da833a26e05c688c94ad", }, }, }, From bcd03d6f82db5fc6e32066f5cab30c90c6a86bd9 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 00:45:24 +0200 Subject: [PATCH 031/142] test(stack): assert every native workload listener binds loopback --- .../whole-stack.native.lifecycle.e2e.test.ts | 12 +- packages/stack/tests/whole-stack/loopback.ts | 149 ++++++++++++++++++ packages/stack/tests/whole-stack/scenarios.ts | 18 +++ packages/stack/tests/whole-stack/workloads.ts | 2 +- 4 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 packages/stack/tests/whole-stack/loopback.ts diff --git a/packages/stack/src/whole-stack.native.lifecycle.e2e.test.ts b/packages/stack/src/whole-stack.native.lifecycle.e2e.test.ts index e71396083e..089f96b2f1 100644 --- a/packages/stack/src/whole-stack.native.lifecycle.e2e.test.ts +++ b/packages/stack/src/whole-stack.native.lifecycle.e2e.test.ts @@ -1,5 +1,10 @@ import { it } from "@effect/vitest"; -import { allEager, defaultLifecycle, writeConfinement } from "../tests/whole-stack/scenarios.ts"; +import { + allEager, + defaultLifecycle, + loopbackOnly, + writeConfinement, +} from "../tests/whole-stack/scenarios.ts"; import { run } from "../tests/whole-stack/fixture.ts"; it.live( @@ -15,3 +20,8 @@ it.live( () => run(writeConfinement("native")), { timeout: 15 * 60_000 }, ); +it.live.skipIf(process.platform === "win32")( + "native: every workload listener binds loopback only", + () => run(loopbackOnly), + { timeout: 15 * 60_000 }, +); diff --git a/packages/stack/tests/whole-stack/loopback.ts b/packages/stack/tests/whole-stack/loopback.ts new file mode 100644 index 0000000000..26f8d08cbc --- /dev/null +++ b/packages/stack/tests/whole-stack/loopback.ts @@ -0,0 +1,149 @@ +import { expect } from "@effect/vitest"; +import { Effect } from "effect"; +import { captureWorkloads, commandOutput } from "./workloads.ts"; +import type { WholeStack } from "./fixture.ts"; + +export type LoopbackViolation = Readonly<{ + readonly pid: number; + readonly address: string; + readonly command: string; +}>; + +type ProcessEntry = Readonly<{ + readonly pid: number; + readonly ppid: number; + readonly command: string; +}>; + +const processTable = Effect.fn("WholeStack.loopback.processTable")(() => + Effect.gen(function* () { + const output = yield* commandOutput("ps", ["-axo", "pid=,ppid=,command="]); + const entries: Array = []; + for (const line of output.split("\n")) { + const match = /^\s*(\d+)\s+(\d+)\s+(.*)$/.exec(line); + if (match === null) continue; + entries.push({ pid: Number(match[1]), ppid: Number(match[2]), command: match[3] ?? "" }); + } + return entries; + }), +); + +const rootWorkloadPids = (identities: ReadonlyArray): ReadonlyArray => + identities + .map((identity) => /^\s*(\d+)\s/.exec(identity)?.[1]) + .filter((value): value is string => value !== undefined) + .map(Number); + +/** Walks the host's process table so a workload's descendants are covered: the native launcher + * spawns each workload into its own separate process group, not the launcher's. */ +const descendantsOf = (roots: ReadonlyArray, entries: ReadonlyArray) => { + const byParent = new Map>(); + for (const entry of entries) { + const list = byParent.get(entry.ppid) ?? []; + list.push(entry.pid); + byParent.set(entry.ppid, list); + } + const seen = new Set(roots); + const queue = [...roots]; + while (queue.length > 0) { + const current = queue.shift(); + if (current === undefined) continue; + for (const child of byParent.get(current) ?? []) + if (!seen.has(child)) { + seen.add(child); + queue.push(child); + } + } + return [...seen]; +}; + +type RawSocket = Readonly<{ readonly pid: number; readonly address: string }>; + +const darwinListeners = Effect.fn("WholeStack.loopback.darwinListeners")( + (pids: ReadonlyArray) => + Effect.gen(function* () { + if (pids.length === 0) return [] as ReadonlyArray; + const output = yield* commandOutput("lsof", [ + "-nP", + "-a", + "-iTCP", + "-sTCP:LISTEN", + "-p", + pids.join(","), + ]); + const sockets: Array = []; + for (const line of output.split("\n").slice(1)) { + const trimmed = line.trim(); + if (trimmed === "") continue; + const fields = trimmed.split(/\s+/); + const pid = Number(fields[1]); + const address = fields.at(-2); + if (!Number.isFinite(pid) || address === undefined) continue; + sockets.push({ pid, address }); + } + return sockets; + }), +); + +const linuxListeners = Effect.fn("WholeStack.loopback.linuxListeners")( + (pids: ReadonlySet) => + Effect.gen(function* () { + const output = yield* commandOutput("ss", ["-ltnpH"]); + const sockets: Array = []; + for (const line of output.split("\n")) { + const trimmed = line.trim(); + if (trimmed === "") continue; + const fields = trimmed.split(/\s+/); + const address = fields[3]; + const processField = fields.at(-1); + const pidMatch = processField === undefined ? null : /pid=(\d+)/.exec(processField); + if (address === undefined || pidMatch === null) continue; + const pid = Number(pidMatch[1]); + if (!pids.has(pid)) continue; + sockets.push({ pid, address }); + } + return sockets; + }), +); + +const hostOf = (address: string): string => { + if (address.startsWith("[")) { + const end = address.indexOf("]"); + return end === -1 ? address : address.slice(1, end); + } + const lastColon = address.lastIndexOf(":"); + return lastColon === -1 ? address : address.slice(0, lastColon); +}; + +const isLoopbackAddress = (address: string): boolean => { + const host = hostOf(address); + return host === "127.0.0.1" || host === "::1" || host === "localhost"; +}; + +/** + * Enumerates every listening TCP socket owned by a native stack's workload process trees (the + * launcher and every descendant it spawns) and reports the ones bound outside loopback. Windows' + * native backend is out of scope; callers skip this on `process.platform === "win32"`. + */ +export const captureLoopbackViolations = Effect.fn("WholeStack.loopback.captureLoopbackViolations")( + (fixture: WholeStack) => + Effect.gen(function* () { + const workloads = yield* captureWorkloads("native", fixture); + const roots = rootWorkloadPids(workloads.identities); + expect(roots.length).toBeGreaterThan(0); + const entries = yield* processTable(); + const pids = descendantsOf(roots, entries); + const commands = new Map(entries.map((entry) => [entry.pid, entry.command] as const)); + const sockets = + process.platform === "linux" + ? yield* linuxListeners(new Set(pids)) + : yield* darwinListeners(pids); + return sockets + .filter((socket) => !isLoopbackAddress(socket.address)) + .map((socket): LoopbackViolation => ({ + pid: socket.pid, + address: socket.address, + command: commands.get(socket.pid) ?? `pid ${socket.pid}`, + })); + }), +); diff --git a/packages/stack/tests/whole-stack/scenarios.ts b/packages/stack/tests/whole-stack/scenarios.ts index ce10ddcf00..2b8faffe7c 100644 --- a/packages/stack/tests/whole-stack/scenarios.ts +++ b/packages/stack/tests/whole-stack/scenarios.ts @@ -28,6 +28,7 @@ import { exerciseAnalytics, queryAnalyticsMarker } from "./analytics.ts"; import { subscribeRealtime } from "./websocket.ts"; import { watchRecoveryMail } from "./realtime-mail.ts"; import { assertWorkloadsGone, captureWorkloads } from "./workloads.ts"; +import { captureLoopbackViolations } from "./loopback.ts"; import { assertOwnerExited, captureOwnerPid } from "../owner.ts"; import { assertConfinedTo, @@ -622,3 +623,20 @@ export const writeConfinement = (runtime: Runtime) => ); }), ); + +/** + * Wakes every service eagerly and drives real traffic through each endpoint, then asserts that no + * listening TCP socket owned by a workload's process tree binds outside loopback. Native only: + * container workloads publish their ports to loopback through Docker's own mapping, a different + * mechanism out of this scenario's scope. + */ +export const loopbackOnly = withFixture("native", (fixture) => + Effect.gen(function* () { + yield* setActivation(fixture.stack, "eager"); + yield* fixture.stack.composition.start; + yield* assertLifecycle(fixture, "running"); + yield* exerciseStack(fixture, "loopback"); + const violations = yield* captureLoopbackViolations(fixture); + expect(violations).toEqual([]); + }), +); diff --git a/packages/stack/tests/whole-stack/workloads.ts b/packages/stack/tests/whole-stack/workloads.ts index da3a1e377d..76546012b2 100644 --- a/packages/stack/tests/whole-stack/workloads.ts +++ b/packages/stack/tests/whole-stack/workloads.ts @@ -10,7 +10,7 @@ export const WorkloadSnapshot = Schema.Struct({ }); export type WorkloadSnapshot = Schema.Schema.Type; -const commandOutput = Effect.fn("WholeStack.commandOutput")( +export const commandOutput = Effect.fn("WholeStack.commandOutput")( (command: string, args: ReadonlyArray) => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; From c7e2f6ef4d6f5134b40411ad0928bd958ef4804e Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 03:47:44 +0200 Subject: [PATCH 032/142] refactor(stack): make the lifecycle reducer the single authority over service state Every lifecycle writer (explicit start, stop, restart and destroy, composition start, traffic admission, idle sleep, configuration changes, storage operations and health/exit observation) now goes through one reducer per owner, applied under a single gate. Commands run outside the gate and report generation-tagged outcomes. - Service keeps only the execution boundary: launch, stop, reprobe and storage for a generation, with confirmed cleanup and a generation fence. - Crashed services stay wakeable behind a breaker; a crash counts once per generation even when its cleanup must be retried. - A failed cleanup stays pending in the reducer, blocking storage and relaunch until a retry confirms it. - Prerequisites stay awake while a dependent runs. A blocked waiter keeps an unhealthy prerequisite reprobed until it recovers or the waiter's budget expires. - Traffic waiters get a 120 s budget from arrival and a 256-waiter cap. - Destroy waits for a running storage operation; new storage, start and restart are refused while it is pending. - Gate permits are acquired and released through the semaphore, so cancellation cannot leak them. --- .../db-config.integration.test.ts | 1 - .../commands/db/dump/dump.integration.test.ts | 1 - .../db/reset/reset.integration.test.ts | 1 - .../generate/generate.integration.test.ts | 1 - .../declarative/sync/sync.integration.test.ts | 1 - .../db/start/start.integration.test.ts | 1 - .../stack/prepare/prepare.integration.test.ts | 1 - .../start-export-pointer.integration.test.ts | 1 - .../stack/start/start.integration.test.ts | 1 - .../stack/status/status.integration.test.ts | 1 - .../serve/serve.stack.integration.test.ts | 1 - apps/cli/tests/helpers/storage.ts | 1 - packages/stack/ARCHITECTURE.md | 209 +- packages/stack/README.md | 4 +- .../stack/src/Commands.integration.test.ts | 6 +- packages/stack/src/Lifecycle.ts | 630 +++++-- packages/stack/src/Lifecycle.unit.test.ts | 438 ++++- .../src/Orchestrator.integration.test.ts | 1582 ++++++++++++---- packages/stack/src/Orchestrator.ts | 1676 ++++++++++------- packages/stack/src/Owner.ts | 147 +- packages/stack/src/Rpc.ts | 1 - .../stack/src/Service.integration.test.ts | 1207 ++++-------- packages/stack/src/Service.ts | 965 ++++------ packages/stack/src/composition/Supabase.ts | 4 +- .../src/composition/Supabase.unit.test.ts | 35 +- ...nalyticsVectorImgproxy.integration.test.ts | 10 +- .../services/AuthStorage.integration.test.ts | 10 +- .../src/services/Database.integration.test.ts | 25 +- .../services/Functions.integration.test.ts | 12 +- .../src/services/Mail.integration.test.ts | 16 +- .../src/services/Pooler.integration.test.ts | 6 +- .../ProcessRecipe.integration.test.ts | 31 +- packages/stack/src/services/ProcessRecipe.ts | 6 + .../src/services/Realtime.integration.test.ts | 10 +- .../src/services/Rest.integration.test.ts | 10 +- .../src/services/Studio.integration.test.ts | 4 +- .../src/services/Vector.integration.test.ts | 10 +- .../stack/src/testing.integration.test.ts | 50 + ...ole-stack.docker.idle-parallel.e2e.test.ts | 8 +- ...ole-stack.native.idle-parallel.e2e.test.ts | 8 +- .../tests/docker-fixture.integration.test.ts | 6 +- packages/stack/tests/standalone-service.ts | 60 + packages/stack/tests/whole-stack/idle.ts | 114 +- packages/stack/tests/whole-stack/scenarios.ts | 3 - 44 files changed, 4400 insertions(+), 2915 deletions(-) create mode 100644 packages/stack/tests/standalone-service.ts diff --git a/apps/cli/src/command-internal/db-config.integration.test.ts b/apps/cli/src/command-internal/db-config.integration.test.ts index 9e8a09e3ae..fd3d0330ec 100644 --- a/apps/cli/src/command-internal/db-config.integration.test.ts +++ b/apps/cli/src/command-internal/db-config.integration.test.ts @@ -460,7 +460,6 @@ describe("dbConfigResolver (db-url under the stack backend)", () => { exit: undefined, currentOperation: undefined, launchId: undefined, - intentRevision: 0, wakeEnabled: state === "running", registered: true, }), diff --git a/apps/cli/src/commands/db/dump/dump.integration.test.ts b/apps/cli/src/commands/db/dump/dump.integration.test.ts index af9f38fc3b..bc36e1d9e5 100644 --- a/apps/cli/src/commands/db/dump/dump.integration.test.ts +++ b/apps/cli/src/commands/db/dump/dump.integration.test.ts @@ -119,7 +119,6 @@ const managedDumpStackApi = (runtime: "native" | "docker") => { exit: undefined, currentOperation: undefined, launchId: undefined, - intentRevision: 0, wakeEnabled: true, registered: true, }), diff --git a/apps/cli/src/commands/db/reset/reset.integration.test.ts b/apps/cli/src/commands/db/reset/reset.integration.test.ts index 4e960d99b1..ee6ab2975d 100644 --- a/apps/cli/src/commands/db/reset/reset.integration.test.ts +++ b/apps/cli/src/commands/db/reset/reset.integration.test.ts @@ -618,7 +618,6 @@ function makeStackObservation( exit: undefined, currentOperation: undefined, launchId: undefined, - intentRevision: 0, wakeEnabled: opts.wakeEnabled ?? false, registered: true, }; diff --git a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts index 7344c9711b..5be13eaaf0 100644 --- a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts @@ -130,7 +130,6 @@ function generateStackApi(workdir: string) { health: "healthy", registered: true, wakeEnabled: true, - intentRevision: 0, currentOperation: undefined, launchId: undefined, exit: undefined, diff --git a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts index 85ebc7eaaa..5c572d651c 100644 --- a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts @@ -132,7 +132,6 @@ function syncStackApi(workdir: string, port: number) { health: "healthy", registered: true, wakeEnabled: true, - intentRevision: 0, currentOperation: undefined, launchId: undefined, exit: undefined, diff --git a/apps/cli/src/commands/db/start/start.integration.test.ts b/apps/cli/src/commands/db/start/start.integration.test.ts index debb4eae72..9c8e1be2c3 100644 --- a/apps/cli/src/commands/db/start/start.integration.test.ts +++ b/apps/cli/src/commands/db/start/start.integration.test.ts @@ -1628,7 +1628,6 @@ describe("db start stack backend", () => { exit: undefined, currentOperation: undefined, launchId: undefined, - intentRevision: 0, wakeEnabled: state.running, registered: true, })), diff --git a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts index cd4a7e1c46..847a5259e1 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts @@ -101,7 +101,6 @@ const makeFixture = (root: string, options: FixtureOptions = {}) => { exit: undefined, currentOperation: undefined, launchId: undefined, - intentRevision: 0, wakeEnabled: false, registered: true, }), diff --git a/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts index ef49fc9753..31396b77a4 100644 --- a/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts @@ -69,7 +69,6 @@ function makeDatabaseStack(sqlPort: number, credentials: StackCredentials): Stac exit: undefined, currentOperation: undefined, launchId: undefined, - intentRevision: 1, wakeEnabled: false, registered: true, }; diff --git a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts index 831e6787eb..bea7a31fdb 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts @@ -114,7 +114,6 @@ const instance = ( exit: undefined, currentOperation: undefined, launchId: undefined, - intentRevision: 0, wakeEnabled: wakeEnabled(), registered: true, }); diff --git a/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts b/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts index ffa4890839..99f2c538db 100644 --- a/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts @@ -106,7 +106,6 @@ const makeObservation = ( exit: undefined, currentOperation: undefined, launchId: undefined, - intentRevision: 1, wakeEnabled: true, registered: true, ...input, diff --git a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts index 45f486e1cd..dced26c90c 100644 --- a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts +++ b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts @@ -93,7 +93,6 @@ const observation = ( cleanupError: undefined, exit: undefined, launchId: 1, - intentRevision: 1, wakeEnabled: false, registered: true, ...overrides, diff --git a/apps/cli/tests/helpers/storage.ts b/apps/cli/tests/helpers/storage.ts index 5a5a1f2479..03933f1982 100644 --- a/apps/cli/tests/helpers/storage.ts +++ b/apps/cli/tests/helpers/storage.ts @@ -160,7 +160,6 @@ const observation = ( exit: undefined, currentOperation: undefined, launchId: undefined, - intentRevision: 1, wakeEnabled, registered: true, }); diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index b03a8b4c2f..92146cade9 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -4,26 +4,27 @@ This document defines the stack package architecture. The package implements the ## Core model -Use **a graph of service instances, with a small serialized lifecycle for each instance**. Keep application readiness separate from that lifecycle. Add a proxy that starts services on public traffic and sleeps them on public inactivity. Run finite commands with the same identity, artifacts and execution primitives, without treating them as services. +Use **a graph of service instances whose lifecycle one reducer per owner decides**. Keep application readiness separate from that lifecycle. Add a proxy that starts services on public traffic and sleeps them on public inactivity. Run finite commands with the same identity, artifacts and execution primitives, without treating them as services. The critical simplification is: > Starting a process and waiting for that process to become healthy are different operations. -A service finishes its current lifecycle transition before accepting another lifecycle signal. There is no cancellation of one transition by another, ownership handoff, special pending-stop state, or general desired-state reconciliation engine. +Lifecycle decisions are pure transitions applied under one gate per owner; executing them is separate. An execution reports generation-tagged outcomes, so a late outcome never affects a newer generation. There is no second lifecycle state outside the reducer. Preserve the useful identity and instance use cases while replacing the underlying machinery. The contract intentionally stays small and behavioral. ## The mental model: a service runs itself; the orchestrator connects services -Establish the individual service lifecycle first, then compose it through a dependency-aware orchestrator. Identity and resource ownership are foundations for both. The orchestrator uses the same service operations for one service or a whole stack; it does not implement a second lifecycle. +Each service execution runs one generation's launch, readiness check and cleanup. The orchestrator's reducer decides when each happens, for one service or a whole stack. Identity and resource ownership are foundations for both. ```mermaid flowchart TB Handle["Public service or stack handle"] --> Orchestrator Proxy["Proxy: traffic and idle signals"] --> Orchestrator Composition["Stack composition
dependency edges and input wiring"] --> Orchestrator - Orchestrator["Orchestrator
validate graph, order operations, await readiness"] --> Executor["Individual service executor
serialize its own lifecycle transitions"] + Orchestrator["Orchestrator
owner gate and lifecycle reducer"] -->|launch, stop, reprobe| Executor["Service execution
one generation's runtime and cleanup"] + Executor -.->|generation-tagged outcomes| Orchestrator Executor --> Backend["Native or container backend
launch, observe, stop, remove"] Context["Shared stack context
identity, owned data, sticky ports"] -.-> Orchestrator Context -.-> Executor @@ -37,10 +38,10 @@ A URL implies a connection requirement, not ownership of another service. Do not | ----------------- | ------------------------------------------------------------------------ | ------------------------------------------------------- | | Recipe/definition | Typed configuration inputs, such as a database URL; own runtime behavior | Where inputs originate or which stack services must run | | Stack composition | Concrete dependency edges and how values populate service inputs | Runtime lifecycle implementation | -| Orchestrator | Graph, input resolution, readiness requirements, selected operations | How PostgreSQL launches or a container stops | -| Service executor | Own state, processes, readiness, data and resolved inputs | Dependency ordering or other services' lifecycle | +| Orchestrator | Graph, lifecycle state, demand, admission, timers and input resolution | How PostgreSQL launches or a container stops | +| Service execution | One generation's processes, readiness check, cleanup and data | When to launch, stop or sleep, or other services' state | -Calling public `service.start()` or `service.stop()` still goes through the orchestrator. The executor is internal, so callers cannot accidentally bypass graph checks. Sleep is an internal proxy policy over the same stop/start mechanics; it has no public service method or RPC endpoint. +Calling public `service.start()` or `service.stop()` goes through the orchestrator. The execution boundary is internal, so callers cannot bypass the reducer. Sleep is an internal idle policy over the same stop/launch mechanics; it has no public service method or RPC endpoint. Each long-running service is an individually identified instance with its own executor, lifecycle and health. There is no capability layer, hidden service group or aggregate service handle. Composition connects instances; it does not hide their identities. @@ -50,7 +51,7 @@ Keep the implementation Effect V4 from the domain inward. Promise is the outer f Organize by cohesive responsibilities. The package shape is: -- `src/` modules: the instance executor, orchestrator, owner, detached host, networking, persistence, and RPC boundary. +- `src/` modules: the lifecycle reducer, orchestrator, service execution, owner, detached host, networking, persistence, and RPC boundary. - `services/`: one definition per service, owning its configuration, endpoints, launch settings, and readiness. `Catalog.ts` validates and dispatches creation; `Recipe.ts` defines their contract and `ProcessRecipe.ts` shares process mechanics. - `composition/Supabase.ts`: default Supabase membership, dependency edges, and input wiring. - `host/`: endpoint projections, command execution, and command attachment transport. @@ -75,20 +76,20 @@ Use that separation for both backends: ```mermaid stateDiagram-v2 - [*] --> Stopped: register - Stopped --> Starting: start or authorized wake - Starting --> Running: process launched - Starting --> Stopped: launch failed and cleanup confirmed - Running --> Stopping: stop, proxy idle sleep, destroy, or unexpected exit + [*] --> Stopped: register (unarmed) + Stopped --> Starting: demand or explicit start + Starting --> Running: session available + Starting --> Failed: launch failed, cleanup confirmed + Running --> Failed: unexpected exit, cleanup confirmed + Running --> Stopping: stop, restart or idle sleep + Starting --> Stopping: stop or restart Stopping --> Stopped: cleanup confirmed + Stopping --> Stopping: cleanup failed, retried on request + Failed --> Starting: demand, behind the breaker + Failed --> Stopped: explicit stop Stopped --> [*]: destroy owned data and registration - note right of Starting - The next lifecycle signal waits. - Preparation already completed. - end note note right of Running - Start transition is complete. Health is observed separately. end note note left of Stopped @@ -97,7 +98,7 @@ stateDiagram-v2 end note ``` -The final marker means the registration no longer exists. `destroy` of a running instance follows stop, then removal, within the admitted operation. If runtime cleanup fails, remain stopping with the current process/exit observation and cleanup error; the next stop/restart retries the remaining cleanup. Do not report a dead process as running or unfinished cleanup as stopped. Failed data removal leaves the registration present. Restart composes stop and launch; it does not add a new stable state. +Each phase after `Stopped` belongs to one generation. The final marker means the registration no longer exists. `destroy` of a running instance stops it, then removes its data under a storage reservation, which it takes once any running storage operation releases it. A generation whose cleanup fails stays stopping, holding its resources; the next stop, restart, start or waiter retries the remaining cleanup. A dead process is never reported as running, nor unfinished cleanup as stopped. Failed data removal leaves the registration present. Restart composes stop and launch; it adds no stable state. ### Chart 2: health does not hold the lifecycle gate @@ -112,13 +113,13 @@ flowchart LR Stopping -->|cleanup confirmed| Stopped["Lifecycle: stopped"] ``` -Only running + healthy satisfies a readiness wait. A health failure records an error and leaves the process running; it neither locks the lifecycle nor automatically stops the process. +Only running + healthy satisfies a readiness wait. A health failure records an error and leaves the process running; it fails the explicit waits that needed it and is re-checked while traffic still waits on it. **For one service instance, running means its process has launched. It does not mean the application is ready.** Every instance exposes its own observation. Separately observe health: starting, healthy, or unhealthy. A failed launch records its error; unsuccessful cleanup must not be reported as a successful stop. Destroy removes the registration only after stopped resources and owned data are safely removed. -Example: PostgreSQL launches, so its start transition finishes in running. Its initialization and health are still pending. A stop request can now run the ordinary stopping transition; there is no start transition to interrupt. A request arriving during actual launch waits for launch to settle first. +Example: PostgreSQL launches, so its generation is running. Its initialization and health are still pending. A stop now interrupts only the readiness check and cleans the session. A stop arriving while the runtime is being acquired waits for that acquisition, then cleans what it produced; a stop arriving during preparation abandons it. -Dependency waits and health waits never hold a lifecycle transition open. Artifact preparation completes before the start transition is admitted, as a shared cache operation whose callers may cancel their own wait. Downloads and image pulls cannot run inside the serialized launch transition. Bounded native pre-launch steps such as initdb remain part of launch. Actual launch and stop work have bounded success/failure paths. A timeout is the current transition settling through its own failure handling, not another command pre-empting it. +Dependency waits and health waits never hold the owner gate or the execution lock. Preparation (artifact downloads and image pulls) runs at the start of each launch, outside the execution lock, and is idempotent. Bounded native pre-launch steps such as initdb remain part of runtime acquisition. Actual launch and stop work have bounded success and failure paths. ## 2. Ownership, instances and composition @@ -227,88 +228,93 @@ flowchart LR The `service: "database"` discriminator selects a service definition and its configuration type; it does not name a group or identify a unique instance. Two databases have the same definition kind and different instance IDs. Operations and dependency edges address instance IDs. Short-lived initialization helpers remain implementation details of an operation; they do not justify a hidden hierarchy of long-running services. -## 3. One lifecycle implementation, one graph planner +## 3. One lifecycle authority per owner -An instance has one serial transition executor. Requests wait until the current transition settles, then revalidate against the resulting state. An already-satisfied request does no runtime work; an explicit stop still updates intent and disables wake as described below. A waiting caller can cancel its wait; an admitted transition belongs to the StackHost and finishes independently of that caller. +Each owner holds one pure reducer, `Lifecycle.ts`: `reduce(state, event, now)` returns the next state and the commands to execute. It owns every service's intent (eager, lazy or stopped), phase (stopped, starting, running, stopping or failed, each tagged with its generation), work leases, waiters, idle epoch, crash-loop breaker, storage reservation and pending destroy. Every lifecycle writer is an event: explicit start, arm, stop, restart and destroy; traffic admission and release; storage reservation; graph updates; and execution outcomes. No other component holds lifecycle state, so status, admission, storage eligibility and retries all read the same record. -### Interruption and command concurrency +### The owner gate -An executing instance operation cannot be interrupted or replaced by another command. Start, stop, restart, destroy, export and restore use the same serial execution gate. Internal proxy-triggered sleep also uses that gate. Incoming commands wait, revalidate against the resulting state, then execute. An instance's restart keeps that gate across stop and launch; readiness observation happens afterward. There is no priority-stop path or ownership handoff. +`Orchestrator.ts` applies events under one gate per owner. One step reduces a batch of events, starts the resulting executions and timers, publishes the new state, then resolves the waiters the step admitted or failed. Executions and timers run outside the gate and report back as events. An event tagged with a generation, idle epoch or breaker cooldown that is no longer current changes nothing, so a late outcome cannot affect a replacement. -```text -start executing → stop requested → stop waits -start settles → stop executes → stopped -``` +| Command | Executed as | +| ----------------------------------- | -------------------------------------------------------------------- | +| `Launch`, `Stop`, `Reprobe` | A fiber in the owner scope driving that service's execution | +| `ArmIdleTimer`, `ArmCooldownTimer` | A timer that reports `IdleElapsed` or `CooldownElapsed` | +| `ArmWaiterTimeout` | The waiting caller's own deadline, which reports `WaiterExpired` | +| `AdmitConnection`, `FailConnection` | Completing that waiter, after the state it was decided on is visible | +| `RequestRejected` | The requesting caller's failure | -Cancellation has a limited meaning at each boundary: +Explicit stop and restart wait on the reducer state itself: a stop completes when its generation leaves `stopping`, and fails while that generation's cleanup is pending. -| Caller activity | Effect of cancellation | -| ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------- | -| Preparing artifacts or waiting for execution admission | Abandon this request before its lifecycle operation begins; shared preparation may continue for other callers | -| Waiting for an executing instance operation | Stop waiting; the StackHost finishes the operation and its cleanup | -| Waiting for readiness or following logs | End the observation without changing lifecycle | -| Running an attached command invocation | Terminate and clean up that invocation | +### The execution boundary -A service is stopped by an explicit stop command, not by a disconnected caller. Launch completes at running, so stop during health-starting uses the ordinary stop operation. If the launch ends while a caller awaits readiness, that observation fails rather than attaching to a later launch. +`Service.ts` executes one generation at a time and owns its resources from before preparation until cleanup is confirmed. A launch prepares, acquires the runtime under the service's execution lock, reports the live session, then runs the readiness check. A stop fences its generation under the same lock, so a launch of that generation that has not acquired its runtime never will; it then interrupts the attempt while it prepares or checks readiness, and cleans only that generation's session, never a newer one. Every outcome is reported: `SessionAvailable`, `LaunchSucceeded`, `ReadinessLost`/`ReadinessRecovered`, `LaunchFailed` and `Exited` once cleanup is confirmed, or `StopFailed` while resources are still held. -A pending orchestration plan is different from an executing transition: stopping REST while its start request waits for database readiness must prevent the old request from later launching REST. The intent revision described below handles this one stale-request check. It does not interrupt an admitted operation. +A generation whose cleanup failed stays `stopping` with its cleanup failure recorded. Nothing launches and no storage operation is admitted while it holds resources, and its waiters fail with the cleanup error. The next explicit stop, restart or start, or the next waiter, retries only the remaining cleanup. -Independent instances execute concurrently. Admission and dependency validation use short coordination sections; downloads, readiness probes, process operations and snapshots do not hold a global execution lock. Each operation settles through success or its own timeout/failure handling and cleanup. Process crashes and owner loss remain possible; report unavailable observations or cleanup errors rather than claiming success. Crash recovery is outside this design. This ownership rule does not require one blanket uninterruptible region around all Effect code. +### Waits and cancellation -### Chart 3: dependency awareness lives in the orchestrator +| Caller activity | Effect of cancellation | +| --------------------------------------- | ---------------------------------------------------------------------- | +| Waiting for the owner gate | Leave without dispatching anything | +| Waiting for traffic admission | Withdraw the waiter; a lease already granted is released | +| Waiting for an explicit operation | Withdraw the wait; the admitted execution continues in the owner scope | +| Waiting for readiness or following logs | End the observation without changing lifecycle | +| Running an attached command invocation | Terminate and clean up that invocation | -Here stack composition has wired the primary database’s connection URL into REST configuration and declared a readiness dependency. The caller composes `start` and `ready`, as a CLI command that needs a usable service would. The service executors never call each other. With an external database URL, the managed database branch is absent. +Traffic waits carry a lease and a wake budget of 120 seconds by default, counted from the client's arrival, including any wait for the gate. At most 256 traffic waiters queue per service; beyond that, admission fails at once. A budget that expires fails only that waiter with the stage it was blocked on; the shared launch continues. Explicit operations (start, ready, restart, composition start) wait without a budget or cap. They fail when their target or a prerequisite fails, loses readiness or fails its cleanup. -```mermaid -sequenceDiagram - participant Caller - participant O as Orchestrator - participant DB as Database executor - participant REST as REST executor - - Caller->>O: Start REST and wait for readiness - O->>O: Read composition edge and input mapping - O->>DB: Start - DB-->>O: Running; health still starting - Note over DB: Launch transition has finished - O->>DB: Await readiness of this launch - Note over O,DB: Observation only; lifecycle remains available - DB-->>O: Healthy - O->>O: Revalidate target intent and graph - O->>REST: Start with ordinary database URL input - REST-->>O: Running; health still starting - O->>REST: Await readiness of this launch - REST-->>O: Healthy - O-->>Caller: REST ready -``` +A service is stopped by an explicit stop, never by a disconnected caller. + +### Demand, admission and readiness -If the prerequisite stops or readiness fails, the wait fails and REST is not launched. The wait does not reserve a long-running start transition. Deliberate stop still goes through the orchestrator's graph checks. +A service has demand while it holds leases or waiters, after an explicit start, while it is eager and has not failed, or while any transitive dependent has demand or is starting, running or stopping. Demand launches a stopped or failed service whose intent is not stopped and whose prerequisites are ready. A failed eager service relaunches only on new demand, so a broken launch is not retried in a loop. -The orchestrator performs a few graph operations: +Admission requires the target and its whole prerequisite closure to be ready. An inspector acquisition needs only a live target session. A readiness failure is not terminal: the session keeps running, and the waiters that awaited that readiness itself, or explicitly through a dependent, fail with its error. While any other waiter is still blocked on it, the reducer keeps re-checking that generation, one check at a time and one second apart after a failed re-check, until it recovers, its generation ends or no waiter needs it; the waiters' budgets bound how long that lasts. Losing a prerequisite never replaces a healthy dependent; a crashed prerequisite with running dependents relaunches because they keep demand on it. -| Operation | Rule | -| --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Start | Start declared prerequisite instances; await their health; launch the selected instance | -| Stop | Admit only when every dependent is stopped with wake disabled, regardless of batch selection; composition stops dependents first | -| Restart | Individual restart follows the same stop check, then launches; composition restart stops in reverse dependency order, then starts dependency-gated members concurrently | -| Sleep | Require proxy inactivity and no running/starting dependent | -| Destroy | Reject while any registered dependent still references the instance; composition/namespace destruction removes dependents first, then performs ordinary stop and owned-data removal | +### Crash-loop breaker -For stop/restart, a running, starting, stopping or wake-armed dependent blocks prerequisite shutdown until that dependent has stopped and settled. Batch membership never bypasses this check. For sleep, a dependent that is already sleeping does not prevent prerequisite sleep. These are admission checks under the same short graph coordination boundary, not long-lived reservations. +Each launch failure or unrequested exit counts once per generation. Three consecutive failures open the breaker for 30 seconds, doubling on each reopening up to 5 minutes. While open, admission fails at once naming the last cause; the cooldown's end re-admits one shared recovery attempt. A generation that stays ready for 30 seconds clears the count. An explicit start, restart or composition start resets the breaker. + +### Chart 3: dependency awareness lives in the reducer + +Here stack composition has wired the primary database's connection URL into REST configuration and declared a readiness dependency. The service executions never call each other. + +```mermaid +sequenceDiagram + participant Caller + participant O as Owner gate and reducer + participant DB as Database execution + participant REST as REST execution + + Caller->>O: StartRequested(REST) and a readiness wait + O->>DB: Launch(generation 1) + Note over O: REST has demand but its prerequisite is not ready + DB-->>O: SessionAvailable, then LaunchSucceeded + O->>REST: Launch(generation 1) with the resolved database URL + REST-->>O: SessionAvailable, then LaunchSucceeded + O-->>Caller: Waiter admitted +``` -Validate composition references, input/output types and cycles when registering or changing the graph. Wiring uses only a source instance ID and named output mapped to a target instance ID and named configuration input. The service catalog declares those input/output schemas. There is no expression language, arbitrary callback or general JSON-path evaluator. Resolve bindings into ordinary configuration before handing it to the service. Validate each service’s configuration separately from that graph. Bind references to immutable instance IDs, not names. Derive ordering from this one graph; do not maintain separate capability and instance lifecycle graphs. +If the database fails, loses readiness or fails its cleanup, the explicit wait fails and REST is not launched. -**Readiness waiting is observation, not a transition.** If PostgreSQL stops while REST is waiting for its health, the wait fails and REST does not launch. It does not restart PostgreSQL or continue using a stale ready result. Readiness is associated with the exact runtime launch. +The reducer applies a few graph rules: -Before launching a dependent, atomically check its prerequisites, verify its own intent revision is unchanged since the request began, and mark its transition starting. Explicit stop advances that revision and disables wake even if no process exists yet; an older request still preparing artifacts or waiting for dependencies cannot launch afterward. This revision fences unadmitted work; it never interrupts an executing transition. This closes the race between dependency validation and another command stopping the prerequisite. Do not reserve an entire composition or acquire a set of batch locks. Each instance operation waits for its own gate and rechecks the current graph and observations on admission. If another caller starts a dependent between two batch steps, stopping the prerequisite is rejected and the batch reports partial results. A stale start plan cannot resume after another operation stops or replaces its prerequisites. +| Operation | Rule | +| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Start | Arm the target and its prerequisites; demand launches each prerequisite, then the target once they are ready | +| Stop | Reject while any transitive dependent has demand, is up or winding down, or is still armed; composition stops dependents first | +| Restart | The same dependent check, then stop and relaunch; composition restart stops in reverse dependency order, then performs composition start | +| Sleep | An armed lazy service sleeps after its idle timeout once it is ready and has no demand, including demand from its dependents | +| Destroy | Reject while a dependency edge still references the instance; then the ordinary stop, a storage reservation for data removal taken once any storage operation releases it, and the graph update removing it. A pending destroy refuses new starts, restarts and storage operations; any other exit gives it up | -Independent instances still execute concurrently. A shadow startup or snapshot must not block Functions restart. Serialize only short in-memory admission decisions and durable writes; never hold the graph decision boundary while downloading, starting processes, probing health, archiving data or writing files. Gateway traffic on unrelated ready instances must not wait behind a state write. +Validate composition references, input/output types and cycles when registering or changing the graph. Wiring uses only a source instance ID and named output mapped to a target instance ID and named configuration input. The service catalog declares those input/output schemas. There is no expression language, arbitrary callback or general JSON-path evaluator. Resolve bindings into ordinary configuration at launch. Bind references to immutable instance IDs, not names. Derive ordering from this one graph. A graph change requires every affected instance to be stopped and unarmed, and the reducer's verdict is checked before the change is persisted. -Composition and namespace methods select instances and call these same operations. Default composition start first assigns/reuses and binds every configured public listener needed by its selected members, including lazy members, and registers their routes. It then starts one dependency-gated worker per selected instance, including prerequisites outside the configured members; independent branches overlap, and outcomes remain in topological order. Eager members launch only after all immediate prerequisites are healthy. Lazy routes arm after their prerequisites settle, including when a prerequisite fails; an eager descendant with a failed prerequisite is reported as blocked and is not launched. It does not select unrelated standalone instances. Successful start returns member observations with their public endpoints; credentials reads render usable connection strings from these assignments without starting lazy backends. It awaits readiness for every launched eager member, including leaves. A failed launch or readiness check makes the operation return an error with per-instance outcomes; a running-but-unhealthy instance remains running, and already completed steps are not rolled back. Armed lazy members need not launch unless required by an eager member. +Composition start first assigns or reuses and binds every configured public listener of its selected members, including lazy members, and registers their routes. One worker per member then waits for its prerequisites' outcomes: a lazy member is armed (also after a prerequisite failed), and an eager member is started and awaited for readiness, or reported as blocked when a prerequisite failed. Prerequisites outside the configured members run as eager standalone instances. It does not select unrelated standalone instances. Successful start returns member observations with their public endpoints; credentials reads render usable connection strings from these assignments without starting lazy backends. A failed launch or readiness check makes the operation return an error with per-instance outcomes; a running but unhealthy instance remains running, and completed steps are not rolled back. After binding the shared API listener, default composition setup uses the endpoint renderer to populate ordinary configuration values: Studio receives `apiUrl` and `publicApiUrl`, Auth receives `externalApiUrl`, and Functions receives `apiUrl`. When the database has a configured SQL endpoint, Functions also receives its rendered `databaseUrl`; this saved value is refreshed when callers recompose the composition, and does not create a database readiness dependency for Function execution. These values are saved in each instance's configuration alongside the saved listener assignment; individual starts and normal host reopening consume that configuration. This is concrete default setup code, not declarative input wiring or a public listener-binding API. The endpoint renderer owns the host/runtime reachability rules; custom configurations supply plain URLs through the same inputs. URL values do not imply graph edges. -Composition restart is two passes: stop the selected instances in reverse dependency order, then perform dependency-gated composition start according to eager/lazy policy. It is not a loop of individual `restart()` calls. If the stop pass fails, report partial results without beginning the start pass. Cancelling composition interrupts its waits and work not yet admitted; already admitted instance operations continue to settle in their owner scope, and launched instances remain until an explicit stop or owner close. Readiness does not hold an instance gate or a composition-wide reservation. Explicit empty selection is a no-op; there is no generic rollback engine. +Composition restart is two passes: stop the selected instances in reverse dependency order, then perform composition start. If the stop pass fails, report partial results without beginning the start pass. Cancelling composition withdraws its waits; admitted executions continue in the owner scope, and launched instances remain until an explicit stop or owner close. Explicit empty selection is a no-op; there is no generic rollback engine. ## 4. Recipes own runtime details @@ -320,9 +326,9 @@ A successful launch returns an owned runtime handle with readiness observation a The database keeps only the initialization information needed to reuse its data on a normal stop/start. Mark initialization complete only after it succeeds. If an ordinary stop interrupts application initialization, the recipe must either support a safe retry or report that initialization is incomplete; do not add a generic persisted operation journal or recovery workflow. -Unexpected process exit is observed by the living host. Immediately disable forwarding and automatic wake for that instance and record its exit result. The executor uses the ordinary serialized stop/remove cleanup for the exact owned runtime handle, reaching stopped only after cleanup succeeds. Stop cannot be a no-op while runtime resources remain; start/restart must finish that cleanup before launching a replacement. If exit occurs during an executing transition, that transition settles and the same gate orders any remaining cleanup; no transition is pre-empted. Old-handle events cannot affect a replacement. This adds no Exited lifecycle state and no cleanup scanner: the host already owns the handle and its exit observation. +Unexpected process exit is observed by the living host. The execution records the exit result, cleans the exact owned session under its execution lock, and reports the exit once cleanup is confirmed, or a cleanup failure while resources remain. Nothing launches a replacement while resources are held. Old-handle events cannot affect a replacement. This adds no cleanup scanner: the host already owns the handle and its exit observation. -There is **no automatic restart, continuous dependency shutdown cascade, or self-healing controller**. A crashed service stays stopped until explicit start/restart. A wake operation may use a running prerequisite or launch a stopped prerequisite whose wake route is armed; an explicitly stopped or unexpectedly exited prerequisite makes wake fail. Only an explicit start may launch those prerequisites. Explicit composition startup can arm initially lazy instances, and sleep retains that permission; unexpected exit removes it. Existing dependents may observe connection failures, as they can with Compose. Deliberate lifecycle changes still obey graph checks. +A crashed service stays armed. Demand relaunches it behind the crash-loop breaker; a crashed prerequisite with running dependents relaunches because they keep demand on it. There is no continuous dependency shutdown cascade: existing dependents may observe connection failures until it is back, as they can with Compose. An explicitly stopped prerequisite makes wake fail. Deliberate lifecycle changes still obey graph checks. ## 5. Sleep stays simple @@ -332,9 +338,9 @@ There is **no automatic restart, continuous dependency shutdown cascade, or self - The idle timer and wake route belong to one instance. Sleep stops that instance after the ordinary dependency checks, retains its data and keeps its wake route armed. It does not implicitly stop companion instances. Instances without a supported public wake route remain running until explicitly stopped; no group sleep mechanism is introduced. - The next request starts the instance through the orchestrator, ensuring its declared prerequisites are ready, then waits for its own health before forwarding. - Explicit stop disables wake. Traffic never reverses it. -- If a prerequisite's sleep is blocked by a running dependent, recheck its existing idle eligibility when that dependent sleeps/stops. No new activity heuristic is needed. +- A prerequisite stays awake while any dependent has demand or is starting, running or stopping. Once the dependent's stop is confirmed, the prerequisite idles on its own timeout. -Sleep is therefore ordinary stop mechanics plus an armed wake route. There is no separate sleep runtime. Activity acquisition and sleep admission must make one atomic in-memory decision so sleep cannot race a request. A request that arrives after sleep was admitted waits for it to finish, then wakes the instance. +Sleep is therefore ordinary stop mechanics plus an armed wake route. There is no separate sleep runtime. The idle timer carries an epoch that any new admission advances, so a request admitted before the reducer commits the sleep keeps the instance running. A request that arrives after the sleep was committed waits for the confirmed stop, then is served by the next generation. ### Chart 4: sleep, stop and destroy have different retained state @@ -353,7 +359,7 @@ flowchart TB Sleep retains the public listener needed to wake. Whole-stack stop closes listeners while preserving saved assignments; the next start rebinds the same ports. Destruction releases only the removed instance's owned assignments, not unrelated services' shared listeners or claims. -Composition validation permits lazy activation only for instances with a configured public wake endpoint. A route-less prerequisite, such as pg-meta without its own public endpoint, is eager; it is not implicitly armed through a dependent. This avoids a second wake-permission mechanism. Internal sleep is available only for instances with a supported public wake route and an enabled idle policy. Database and Functions need no automatic idle timer by default. Functions inspector access remains possible while health is starting; ordinary application traffic waits for healthy. +Composition validation permits lazy activation only for instances with a configured public wake endpoint. A route-less prerequisite, such as pg-meta without its own public endpoint, is eager; it is not implicitly armed through a dependent. This avoids a second wake-permission mechanism. Internal sleep is available only for lazy instances with a supported public wake route and an enabled idle policy. The default composition gives every lazy service one, Functions included; the database is eager. Functions inspector access remains possible while health is starting; ordinary application traffic waits for healthy. ## 6. Commands share execution, not service lifecycle @@ -516,16 +522,16 @@ Once an instance operation starts executing, the host owns it until settlement. sequenceDiagram participant Client participant RPC as RPC handler - participant Executor as Host-owned executor + participant Executor as Owner gate and executions Client->>RPC: service.start(id) - RPC->>Executor: Submit start - Executor->>Executor: Admit and execute launch + RPC->>Executor: StartRequested and a readiness wait + Executor->>Executor: Reduce and launch in the owner scope Client--xRPC: Client disconnects - Note over RPC: Caller stops waiting - Note over Executor: Executing operation continues - Executor->>Executor: Record running or failure + Note over RPC: Caller's wait is withdrawn + Note over Executor: Launch continues + Executor->>Executor: Reduce the reported outcome Client->>RPC: Reconnect; service.status(id) - RPC->>Executor: Read observation + RPC->>Executor: Read status Executor-->>RPC: Current state RPC-->>Client: Current state ``` @@ -536,16 +542,17 @@ Attached commands have a separate contract. `runCommand` streams `started(jobId) ## 8. Keep safety infrastructure at its boundary -### Instance observations are the runtime source of truth +### The reducer state is the lifecycle source of truth -Each instance owns one observable runtime state. Its executor updates lifecycle and current operation; runtime observations update health, exits and errors. `service.status`, `service.followStatus`, dependency readiness checks and CLI presentation consume those same observations. Endpoint availability comes from the component that owns the listener. The StackHost transports these observations without maintaining another lifecycle copy. +Lifecycle and health come from the owner's reducer state; the service execution adds only execution facts: configuration, the last error, a pending cleanup error, the exit result and the current operation. `service.status` and `service.followStatus` join the two, and admission, dependency readiness and CLI presentation read the same state. Endpoint availability comes from the component that owns the listener. The StackHost transports these observations without maintaining another lifecycle copy. ```mermaid flowchart LR - Executor["Executor transitions"] --> Instance["Instance observation"] - Runtime["Health and process events"] --> Instance - Instance --> RPC["status and followStatus"] - Instance --> Dependencies["Orchestrator readiness checks"] + Outcomes["Execution outcomes and requests"] --> Reducer["Reducer state"] + Execution["Execution facts"] --> Status["Instance status"] + Reducer --> Status + Reducer --> Admission["Admission and readiness"] + Status --> RPC["status and followStatus"] RPC --> CLI["CLI display"] ``` @@ -682,7 +689,7 @@ Keep the contract narrow: - Save requires confirmed stopped, initialized data. The caller explicitly stops first; snapshotting does not secretly stop dependents or restart services. A save publishes only a complete entry and replaces the previous entry for the same key. - Restore requires a confirmed stopped instance with empty data. Validate format, artifact/runtime compatibility and initialization profile before installing restored data. A missing key returns `false`; a compatible published entry returns `true`; reject a nonempty target rather than overwriting it. -- Both operations occupy the instance's existing serial operation gate and leave lifecycle stopped. Queued start, destroy or another storage operation waits for settlement and revalidates. No new lifecycle states are necessary; the observable pending operation identifies snapshot work. An armed wake route is not a substitute for explicit stop. +- Both operations hold the instance's storage reservation in the lifecycle reducer and leave it stopped. While the reservation is held, a start keeps the instance stopped until it is released and another storage operation is rejected at once. A destroy waits for the storage operation to settle, then takes the reservation; once a destroy is pending, new storage operations are refused, so it cannot be starved. No new lifecycle states are necessary; the observable current operation identifies snapshot work. An armed wake route is not a substitute for explicit stop. - Native snapshots copy or clone the host data; container snapshots copy database data through a managed volume and helper. Docker data normally lives in a managed volume, while existing host data can be retained through the host-backed fallback. The host storage marker detects a missing or mismatched Docker volume; deleting that volume loses its database data. - Restore transfers compatible database contents, not the source instance's identity, public port claims or composition membership. The target retains its own data location and configuration, with database-specific credentials reconciled before readiness. Cache snapshots survive destruction of the source instance because their managed storage is separate; instance snapshots restore only into their own instance. @@ -692,7 +699,7 @@ These are physical database snapshots for the cache use case. A `pg_dump` invoca `DatabaseInstance.resetData` removes the selected database instance's owned data and initialization metadata while retaining its registration, configuration, composition bindings, and public port assignments. It is database-specific, alongside snapshot save and restore; other service types do not expose a reset operation. -The caller must stop the database with wake disabled first. Reset runs through the same serialized storage-operation gate as snapshots and leaves the database stopped. Deletion uses the database's ownership checks and runtime-specific filesystem handling. The next normal start initializes a fresh PostgreSQL cluster using the retained configuration. Reset does not apply project migrations or seeds, stop other services, or resume the composition; those decisions belong to the CLI. It adds no lifecycle state or persisted recovery phase. +The caller must stop the database with wake disabled first. Reset holds the same storage reservation as snapshots and leaves the database stopped. Deletion uses the database's ownership checks and runtime-specific filesystem handling. The next normal start initializes a fresh PostgreSQL cluster using the retained configuration. Reset does not apply project migrations or seeds, stop other services, or resume the composition; those decisions belong to the CLI. It adds no lifecycle state or persisted recovery phase. ## 9. What changes for the CLI @@ -710,7 +717,7 @@ Make lifecycle and health explicit in status instead of forcing them into the ol | Process exited | Exit result is visible while ordinary cleanup settles, then stopped; report unexpected exit as failure | | Stop cleanup failed | Runtime cleanup remains stopping with current process/exit observation and error; stop/restart retries cleanup | -Functions serve observes lifecycle and health separately and reports an unhealthy or stopped runtime. DB/storage readiness checks require running and healthy. The current operation remains observable in memory, but shadow finalizers no longer poll it: `destroy()` queues behind an executing snapshot and runs after settlement. Do not persist it for crash recovery. Health observations are attached to the current launch; they cannot make a replacement healthy or failed. +Functions serve observes lifecycle and health separately and reports an unhealthy or stopped runtime. DB/storage readiness checks require running and healthy. The current operation remains observable in memory. `destroy()` issued during a snapshot waits for it to settle, then removes the data. Do not persist it for crash recovery. Health observations are attached to the current launch; they cannot make a replacement healthy or failed. The public surface stays recognizable: create/open/discover, services create/get/list, lifecycle, credentials, status/logs and preparation. Promise stays an outer adapter over Effect. The important API clarification is **launch versus readiness**. diff --git a/packages/stack/README.md b/packages/stack/README.md index 5bf92c57b4..6b7cb62f3a 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -144,7 +144,7 @@ const services = await stack.composition.supabase([ await stack.composition.start(); ``` -The factory accepts one instance of each selected recipe, binds its configured public endpoints, and wires managed inputs such as REST's database URL. When the database SQL endpoint is configured, Functions receives the saved database URL as an ordinary input too; recompose the composition after rotating database credentials to refresh that value. This binding does not make Functions wait for database readiness. Database is eager; Functions are lazy without an idle timeout; Studio is lazy with a 5-minute idle timeout; other public services are lazy with a 60-second idle timeout. Services without public endpoints are eager. Pass `{ eager: true }` to start every member eagerly. A managed URL binding requires its producer's endpoint to be configured. The factory also supplies ordinary host/runtime API URLs to Auth, Studio, and Functions without adding dependencies from those URLs. It rejects an already configured composition. +The factory accepts one instance of each selected recipe, binds its configured public endpoints, and wires managed inputs such as REST's database URL. When the database SQL endpoint is configured, Functions receives the saved database URL as an ordinary input too; recompose the composition after rotating database credentials to refresh that value. This binding does not make Functions wait for database readiness. Database is eager; Studio is lazy with a 5-minute idle timeout; other public services, Functions included, are lazy with a 60-second idle timeout. Services without public endpoints are eager. Pass `{ eager: true }` to start every member eagerly. A managed URL binding requires its producer's endpoint to be configured. The factory also supplies ordinary host/runtime API URLs to Auth, Studio, and Functions without adding dependencies from those URLs. It rejects an already configured composition. Inputs that the composition binds or the owner fills from the stack credentials, such as REST's `databaseUrl` or a JWT secret, are optional in creation configuration. Starting or restarting an instance without a required input that was neither bound nor provided fails before any lifecycle change with a `ServiceError` whose `operation` is `"input"` and whose message names the input; a running instance keeps running. @@ -177,7 +177,7 @@ await stack.composition.configure({ await stack.composition.start(); ``` -Bindings supply ordinary configuration values; a URL alone never creates a dependency. An independent REST instance can instead receive an external database URL. Lazy members receive public listeners before their processes start. Public traffic starts an armed instance and waits for health; inspector connections can attach before health succeeds. Explicit stop disables wake. An individual restart runs the instance explicitly; restart the composition to reapply its lazy policy. +Bindings supply ordinary configuration values; a URL alone never creates a dependency. An independent REST instance can instead receive an external database URL. Lazy members receive public listeners before their processes start. Public traffic starts an armed instance and waits for health, up to a 120-second wake budget; inspector connections can attach before health succeeds. A prerequisite stays awake while any dependent runs. A crashed instance stays armed and the next request wakes it again; after three consecutive failures it fails fast for a cooldown of 30 seconds, doubling up to 5 minutes, until the cooldown elapses or an explicit start. Explicit stop disables wake. An individual restart runs the instance explicitly; restart the composition to reapply its lazy policy. - Instance methods affect that instance, subject to dependency checks. - Composition methods affect selected members; startup also includes declared prerequisites. diff --git a/packages/stack/src/Commands.integration.test.ts b/packages/stack/src/Commands.integration.test.ts index 62eadbcdef..bef7989f20 100644 --- a/packages/stack/src/Commands.integration.test.ts +++ b/packages/stack/src/Commands.integration.test.ts @@ -22,7 +22,7 @@ import * as CommandRunner from "./host/CommandRunner.ts"; import { CommandError } from "./host/CommandRunner.ts"; import type { PgProveOptions, PostgresCommand } from "./Commands.ts"; import { makeDatabase } from "./services/Database.ts"; -import { makeService } from "./Service.ts"; +import { makeStandaloneService } from "../tests/standalone-service.ts"; import { bindTcp, serveTcp } from "./Proxy.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; import { noContainerClaims, noDirectoryClaims } from "../tests/claims.ts"; @@ -99,7 +99,7 @@ describe("finite PostgreSQL commands", { timeout: 180_000 }, () => { runtime, ...(runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), }); - const service = yield* makeService(database.definition, { + const service = yield* makeStandaloneService(database.definition, { id: "database", config: { version: "17", @@ -252,7 +252,7 @@ describe("finite PostgreSQL commands", { timeout: 180_000 }, () => { runtime, ...(runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), }); - const service = yield* makeService(database.definition, { + const service = yield* makeStandaloneService(database.definition, { id: "database", config: { version: String(major), diff --git a/packages/stack/src/Lifecycle.ts b/packages/stack/src/Lifecycle.ts index 58cfa84f65..ff88493097 100644 --- a/packages/stack/src/Lifecycle.ts +++ b/packages/stack/src/Lifecycle.ts @@ -7,6 +7,8 @@ const breakerThreshold = 3; const breakerBaseCooldownMillis = 30_000; const breakerMaxCooldownMillis = 300_000; const breakerStabilityMillis = 30_000; +/** The pause between a failed re-check of a blocking session and the next one. */ +const reprobeSpacingMillis = 1_000; /** One member of the owner's composition graph, as the reducer needs to see it. */ export interface ServiceSpec { @@ -114,9 +116,12 @@ const initialBreaker: BreakerState = { interface Waiter { readonly id: number; - readonly deadline: number; + /** Undefined for an explicit operation's wait, which has no wake budget. */ + readonly deadline: number | undefined; /** False for an acquisition (such as the Functions inspector) that bypasses target readiness. */ readonly requireReady: boolean; + /** Whether admission opens a work lease; an explicit operation's wait only observes readiness. */ + readonly lease: boolean; } export interface ServiceState { @@ -131,8 +136,25 @@ export interface ServiceState { readonly waiters: ReadonlyMap; /** Set by an explicit start or restart so the next launch attempt proceeds without demand. */ readonly relaunchForced: boolean; - /** Blocks every launch while a storage operation owns the stopped service. */ + /** Blocks every launch while a storage operation or destroy owns the stopped service. */ readonly storageReserved: boolean; + /** + * A destroy waiting for the service to stop and any storage operation to release + * (`requested`), or holding the storage reservation to remove its data (`reserved`). + */ + readonly destroy: "requested" | "reserved" | undefined; + /** + * The current generation's last failed readiness check, cleared once it recovers or ends. + * `reprobed` marks a failure that a re-check already confirmed, so the next one is spaced out. + */ + readonly readinessFailure: { readonly cause: unknown; readonly reprobed: boolean } | undefined; + /** Whether a `Reprobe` of the current generation is in flight; only one runs at a time. */ + readonly reprobing: boolean; + /** + * Set while `Stopping` when the last cleanup of that generation failed: its resources are still + * held and no stop is in flight. A stop, restart, start or new waiter retries the cleanup. + */ + readonly cleanupFailure: { readonly cause: unknown } | undefined; } const initialService = (spec: ServiceSpec, now: number): ServiceState => ({ @@ -146,6 +168,10 @@ const initialService = (spec: ServiceSpec, now: number): ServiceState => ({ waiters: new Map(), relaunchForced: false, storageReserved: false, + destroy: undefined, + readinessFailure: undefined, + reprobing: false, + cleanupFailure: undefined, }); export interface LifecycleState { @@ -170,13 +196,23 @@ export const initialState = (graph: LifecycleGraph, now: number): LifecycleState }); export type LifecycleEvent = Data.TaggedEnum<{ - /** `requireReady: false` lets an acquisition (the Functions inspector) bypass target readiness. */ + /** + * `requireReady: false` lets an acquisition (the Functions inspector) bypass target readiness. + * `openedAt` starts the wake budget when the client arrived, not when the gate admitted it. + */ ConnectionOpened: { readonly id: string; readonly waiterId: number; readonly requireReady: boolean; + readonly openedAt?: number; }; ConnectionClosed: { readonly id: string }; + /** An explicit operation waiting for a session (or readiness) with no budget and no lease. */ + ReadinessAwaited: { + readonly id: string; + readonly waiterId: number; + readonly requireReady: boolean; + }; /** A live session exists but hasn't passed its first health check; admits `requireReady: false`. */ SessionAvailable: { readonly id: string; readonly generation: number }; LaunchSucceeded: { readonly id: string; readonly generation: number }; @@ -187,6 +223,17 @@ export type LifecycleEvent = Data.TaggedEnum<{ readonly cause: unknown; readonly requested: boolean; }; + /** + * A stop or a failed launch's cleanup of `generation` failed; its resources are still held. + * `failure` carries the launch failure or crash that ended the generation, if one did, so it + * still counts toward the breaker once. + */ + StopFailed: { + readonly id: string; + readonly generation: number; + readonly cause: unknown; + readonly failure?: unknown; + }; IdleElapsed: { readonly id: string; readonly generation: number; readonly epoch: number }; ReadinessLost: { readonly id: string; readonly generation: number; readonly cause: unknown }; ReadinessRecovered: { readonly id: string; readonly generation: number }; @@ -194,13 +241,22 @@ export type LifecycleEvent = Data.TaggedEnum<{ /** Fired by the runtime once a breaker's cooldown elapses; stale against the current breaker is a no-op. */ CooldownElapsed: { readonly id: string; readonly openUntil: number }; StartRequested: { readonly id: string }; + /** Arms a service for demand-driven launches without forcing one. */ + ArmRequested: { readonly id: string }; StopRequested: { readonly id: string }; RestartRequested: { readonly id: string }; StorageReserved: { readonly id: string }; StorageReleased: { readonly id: string }; + /** Stops the service and reserves its storage for data removal once both are free. */ + DestroyRequested: { readonly id: string }; + /** A destroy gives up its request or reservation, whether or not it removed the data. */ + DestroyReleased: { readonly id: string }; WaiterExpired: { readonly id: string; readonly waiterId: number }; WaiterCancelled: { readonly id: string; readonly waiterId: number }; - /** Atomically swaps the graph while preserving every continuing service's generation and epoch. */ + /** + * Atomically swaps the graph while preserving every continuing service's generation and epoch. + * A service the update adds starts unarmed, so configuring a graph never launches anything. + */ GraphUpdated: { readonly graph: LifecycleGraph }; }>; export const LifecycleEvent = Data.taggedEnum(); @@ -221,8 +277,8 @@ export type LifecycleCommand = Data.TaggedEnum<{ readonly openUntil: number; readonly delayMillis: number; }; - /** Re-run the live session's readiness probe after it lost readiness without exiting. */ - Reprobe: { readonly id: string; readonly generation: number }; + /** Re-runs a live session's failed readiness check, after `delayMillis`, because a waiter needs it. */ + Reprobe: { readonly id: string; readonly generation: number; readonly delayMillis: number }; AdmitConnection: { readonly id: string; readonly waiterId: number }; FailConnection: { readonly id: string; @@ -230,10 +286,10 @@ export type LifecycleCommand = Data.TaggedEnum<{ readonly message: string; readonly cause: unknown; }; - /** An explicit stop, restart, storage reservation or graph update the reducer refused to admit. */ + /** An explicit start, stop, restart, storage reservation or graph update the reducer refused. */ RequestRejected: { readonly id: string; - readonly operation: "stop" | "restart" | "storage" | "graph"; + readonly operation: "start" | "stop" | "restart" | "storage" | "graph"; readonly message: string; }; }>; @@ -254,8 +310,12 @@ const isUpOrWindingDown = (phase: Phase): boolean => const isReady = (phase: Phase): boolean => phase._tag === "Running" && phase.ready; const isSessionAvailable = (phase: Phase): boolean => phase._tag === "Running"; +/** + * An eager service's standing demand lapses once it fails, so a broken launch isn't retried in a + * loop; traffic, a running dependent or an explicit start still relaunch it. + */ const ownsDemand = (service: ServiceState): boolean => - service.intent === "eager" || + (service.intent === "eager" && service.phase._tag !== "Failed") || service.leases > 0 || service.waiters.size > 0 || service.relaunchForced; @@ -393,6 +453,125 @@ const failAllWaiters = ( return { ...service, waiters: new Map() }; }; +/** + * Fails every explicit operation's wait on a transitive dependent of a prerequisite that just + * failed or lost readiness: such a wait has no budget, so it must end with the prerequisite's + * outcome. Traffic waiters keep their demand and stay bounded by their own budget. + */ +const failExplicitDependentWaiters = ( + state: LifecycleState, + id: string, + message: string, + cause: unknown, + commands: Array, +): LifecycleState => { + let next = state; + for (const dependent of state.graph.transitiveDependents.get(id) ?? []) { + const service = next.services.get(dependent); + if (service === undefined) continue; + const explicit = [...service.waiters.values()].filter((waiter) => !waiter.lease); + if (explicit.length === 0) continue; + const waiters = new Map(service.waiters); + for (const waiter of explicit) { + waiters.delete(waiter.id); + commands.push( + LifecycleCommand.FailConnection({ id: dependent, waiterId: waiter.id, message, cause }), + ); + } + next = setService(next, dependent, (s) => ({ ...s, waiters })); + } + return next; +}; + +/** Re-issues the stop of a generation whose last cleanup failed, if one is pending. */ +const retryCleanup = ( + state: LifecycleState, + id: string, + commands: Array, +): LifecycleState => { + const service = state.services.get(id); + if (service?.phase._tag !== "Stopping" || service.cleanupFailure === undefined) return state; + commands.push(LifecycleCommand.Stop({ id, generation: service.phase.generation })); + return setService(state, id, (s) => ({ ...s, cleanupFailure: undefined })); +}; + +/** + * Retries a failed cleanup that a newly queued waiter is blocked on, in its prerequisite closure + * or the target itself. Only a new arrival retries, so a cleanup that keeps failing never loops. + */ +const retryBlockedCleanup = ( + state: LifecycleState, + id: string, + commands: Array, +): LifecycleState => { + let next = state; + for (const blocker of [...(state.graph.prerequisiteClosure.get(id) ?? []), id]) + next = retryCleanup(next, blocker, commands); + return next; +}; + +/** + * Whether a pending waiter needs this service ready: one of its own that requires readiness, or + * any waiter of a transitive dependent, since admission requires the whole prerequisite closure. + */ +const blocksWaiters = (state: LifecycleState, id: string): boolean => { + const service = state.services.get(id); + if (service === undefined) return false; + for (const waiter of service.waiters.values()) if (waiter.requireReady) return true; + for (const dependent of state.graph.transitiveDependents.get(id) ?? []) + if ((state.services.get(dependent)?.waiters.size ?? 0) > 0) return true; + return false; +}; + +/** A service's wake budget; a traffic waiter's deadline counts from when the client arrived. */ +export const waiterBudgetMillis = (graph: LifecycleGraph, id: string): number => + graph.services.get(id)?.waiterBudgetMillis ?? defaultWaiterBudgetMillis; + +/** Queues or immediately resolves one traffic acquisition or explicit readiness wait. */ +const openWaiter = ( + state: LifecycleState, + id: string, + waiterId: number, + requireReady: boolean, + traffic: boolean, + now: number, + openedAt = now, +): readonly [LifecycleState, ReadonlyArray] => { + const service = state.services.get(id); + const spec = state.graph.services.get(id); + if (service === undefined || spec === undefined) return [state, []]; + const fail = (message: string, cause: unknown) => + [state, [LifecycleCommand.FailConnection({ id, waiterId, message, cause })]] as const; + if (service.intent === "stopped") return fail(`${id} is explicitly stopped`, undefined); + const eligibleNow = requireReady ? admissionReady(state, id) : sessionReady(state, id); + if (eligibleNow) + return [ + traffic + ? setService(state, id, (s) => ({ ...s, leases: s.leases + 1, lastActivity: now })) + : state, + [LifecycleCommand.AdmitConnection({ id, waiterId })], + ]; + if (isBreakerOpen(service.breaker)) + return fail(`${id} circuit breaker is open`, service.breaker.lastCause); + // The cap bounds sockets held by traffic; explicit operations are few and already bounded. + const trafficWaiters = [...service.waiters.values()].filter((waiter) => waiter.lease).length; + if (traffic && trafficWaiters >= waiterCap) return fail(`${id} has too many waiters`, undefined); + const deadline = traffic ? openedAt + waiterBudgetMillis(state.graph, id) : undefined; + const commands: Array = + deadline === undefined ? [] : [LifecycleCommand.ArmWaiterTimeout({ id, waiterId, deadline })]; + const queued = setService(state, id, (s) => ({ + ...s, + lastActivity: traffic ? now : s.lastActivity, + waiters: new Map(s.waiters).set(waiterId, { + id: waiterId, + deadline, + requireReady, + lease: traffic, + }), + })); + return [retryBlockedCleanup(queued, id, commands), commands]; +}; + /** The transitive dependents currently active enough to block a stop or restart of a prerequisite. */ const activeDependents = (state: LifecycleState, id: string): ReadonlyArray => [...(state.graph.transitiveDependents.get(id) ?? [])].filter((dependentId) => { @@ -400,6 +579,60 @@ const activeDependents = (state: LifecycleState, id: string): ReadonlyArray, +): LifecycleState | undefined => { + const service = state.services.get(id); + if (service === undefined) return undefined; + // Checked independently of whether this service itself needs a `Stop` command: a `Failed` + // target with an open breaker must still protect a dependent's retained demand. + const blockers = activeDependents(state, id); + if (blockers.length > 0) { + commands.push( + LifecycleCommand.RequestRejected({ + id, + operation: "stop", + message: `${id} has active dependents: ${blockers.join(", ")}`, + }), + ); + return undefined; + } + const winding = isUpOrWindingDown(service.phase) && service.phase._tag !== "Stopping"; + const failed = failAllWaiters(service, id, `${id} was explicitly stopped`, undefined, commands); + const generation = generationOf(failed.phase); + if (winding && generation !== undefined) commands.push(LifecycleCommand.Stop({ id, generation })); + const phase = + winding && generation !== undefined + ? Phase.Stopping({ generation }) + : failed.phase._tag === "Failed" + ? Phase.Stopped() + : failed.phase; + const stopped = setService(state, id, () => ({ + ...failed, + intent: "stopped", + relaunchForced: false, + phase, + })); + return retryCleanup(stopped, id, commands); +}; + +/** Rejects an explicit start or restart of a service a destroy has claimed. */ +const rejectWhileDestroying = ( + service: ServiceState, + id: string, + operation: "start" | "restart", + commands: Array, +): boolean => { + if (service.destroy === undefined) return false; + commands.push( + LifecycleCommand.RequestRejected({ id, operation, message: `${id} is being destroyed` }), + ); + return true; +}; + /** Re-arms every prerequisite an explicit start or restart depends on, clearing a stopped intent. */ const rearmPrerequisites = (state: LifecycleState, id: string): LifecycleState => { let next = state; @@ -414,6 +647,39 @@ const rearmPrerequisites = (state: LifecycleState, id: string): LifecycleState = }; /** Shared breaker/phase bookkeeping for a launch failure or an unrequested exit. */ +/** Counts one generation's launch failure or crash, opening the breaker at the threshold. */ +const countFailure = ( + service: ServiceState, + id: string, + cause: unknown, + now: number, + commands: Array, +): BreakerState => { + const materialized = materializeStability(service.breaker, service.phase, now); + const consecutiveFailures = materialized.consecutiveFailures + 1; + if (consecutiveFailures < breakerThreshold) + return { + consecutiveFailures, + openUntil: undefined, + cooldownMillis: materialized.cooldownMillis, + lastCause: cause, + stableGeneration: undefined, + stableSince: undefined, + }; + const openUntil = now + materialized.cooldownMillis; + commands.push( + LifecycleCommand.ArmCooldownTimer({ id, openUntil, delayMillis: materialized.cooldownMillis }), + ); + return { + consecutiveFailures, + openUntil, + cooldownMillis: Math.min(breakerMaxCooldownMillis, materialized.cooldownMillis * 2), + lastCause: cause, + stableGeneration: undefined, + stableSince: undefined, + }; +}; + const recordFailure = ( state: LifecycleState, id: string, @@ -422,47 +688,22 @@ const recordFailure = ( now: number, commands: Array, ): LifecycleState => - setService(state, id, (service) => { - const materialized = materializeStability(service.breaker, service.phase, now); - const consecutiveFailures = materialized.consecutiveFailures + 1; - const opens = consecutiveFailures >= breakerThreshold; - let breaker: BreakerState; - if (opens) { - const openUntil = now + materialized.cooldownMillis; - commands.push( - LifecycleCommand.ArmCooldownTimer({ - id, - openUntil, - delayMillis: materialized.cooldownMillis, - }), - ); - breaker = { - consecutiveFailures, - openUntil, - cooldownMillis: Math.min(breakerMaxCooldownMillis, materialized.cooldownMillis * 2), - lastCause: cause, - stableGeneration: undefined, - stableSince: undefined, + setService( + failExplicitDependentWaiters(state, id, `prerequisite ${id} failed`, cause, commands), + id, + (service) => { + const failed = failAllWaiters(service, id, `${id} failed`, cause, commands); + return { + ...failed, + phase: Phase.Failed({ generation, cause }), + breaker: countFailure(service, id, cause, now, commands), + idleArmedEpoch: undefined, + relaunchForced: false, + readinessFailure: undefined, + reprobing: false, }; - } else { - breaker = { - consecutiveFailures, - openUntil: undefined, - cooldownMillis: materialized.cooldownMillis, - lastCause: cause, - stableGeneration: undefined, - stableSince: undefined, - }; - } - const failed = failAllWaiters(service, id, `${id} failed`, cause, commands); - return { - ...failed, - phase: Phase.Failed({ generation, cause }), - breaker, - idleArmedEpoch: undefined, - relaunchForced: false, - }; - }); + }, + ); const applyEvent = ( state: LifecycleState, @@ -472,62 +713,18 @@ const applyEvent = ( const commands: Array = []; switch (event._tag) { - case "ConnectionOpened": { - const { id, waiterId, requireReady } = event; - const service = state.services.get(id); - const spec = state.graph.services.get(id); - if (service === undefined || spec === undefined) break; - if (service.intent === "stopped") { - commands.push( - LifecycleCommand.FailConnection({ - id, - waiterId, - message: `${id} is explicitly stopped`, - cause: undefined, - }), - ); - break; - } - const eligibleNow = requireReady ? admissionReady(state, id) : sessionReady(state, id); - if (eligibleNow) { - return [ - setService(state, id, (s) => ({ ...s, leases: s.leases + 1, lastActivity: now })), - [LifecycleCommand.AdmitConnection({ id, waiterId })], - ]; - } - if (isBreakerOpen(service.breaker)) { - commands.push( - LifecycleCommand.FailConnection({ - id, - waiterId, - message: `${id} circuit breaker is open`, - cause: service.breaker.lastCause, - }), - ); - break; - } - if (service.waiters.size >= waiterCap) { - commands.push( - LifecycleCommand.FailConnection({ - id, - waiterId, - message: `${id} has too many waiters`, - cause: undefined, - }), - ); - break; - } - const deadline = now + (spec.waiterBudgetMillis ?? defaultWaiterBudgetMillis); - commands.push(LifecycleCommand.ArmWaiterTimeout({ id, waiterId, deadline })); - return [ - setService(state, id, (s) => ({ - ...s, - lastActivity: now, - waiters: new Map(s.waiters).set(waiterId, { id: waiterId, deadline, requireReady }), - })), - commands, - ]; - } + case "ConnectionOpened": + return openWaiter( + state, + event.id, + event.waiterId, + event.requireReady, + true, + now, + event.openedAt, + ); + case "ReadinessAwaited": + return openWaiter(state, event.id, event.waiterId, event.requireReady, false, now); case "ConnectionClosed": { return [ setService(state, event.id, (s) => ({ @@ -551,6 +748,8 @@ const applyEvent = ( setService(state, id, (s) => ({ ...s, phase: Phase.Running({ generation, ready: false }), + readinessFailure: undefined, + reprobing: false, })), commands, ]; @@ -568,6 +767,8 @@ const applyEvent = ( ...s, phase: Phase.Running({ generation, ready: true }), breaker: { ...s.breaker, stableGeneration: generation, stableSince: now }, + readinessFailure: undefined, + reprobing: false, })), commands, ]; @@ -593,6 +794,9 @@ const applyEvent = ( ...s, phase: Phase.Stopped(), idleArmedEpoch: undefined, + readinessFailure: undefined, + reprobing: false, + cleanupFailure: undefined, })), commands, ]; @@ -601,6 +805,39 @@ const applyEvent = ( return [recordFailure(state, id, generation, cause, now, commands), commands]; break; } + case "StopFailed": { + const { id, generation, cause, failure } = event; + const service = state.services.get(id); + if (service === undefined || generationOf(service.phase) !== generation) break; + // Nothing may launch or reserve storage while resources are held, and waiters can't be + // served by a generation that is going away, so they get the cleanup failure now. + const message = `${id} cleanup failed`; + const dependentsFailed = failExplicitDependentWaiters( + state, + id, + `prerequisite ${message}`, + cause, + commands, + ); + return [ + setService(dependentsFailed, id, (s) => ({ + ...failAllWaiters(s, id, message, cause, commands), + phase: Phase.Stopping({ generation }), + breaker: + failure === undefined || + service.phase._tag === "Stopping" || + service.phase._tag === "Failed" + ? s.breaker + : countFailure(s, id, failure, now, commands), + idleArmedEpoch: undefined, + relaunchForced: false, + readinessFailure: undefined, + reprobing: false, + cleanupFailure: { cause }, + })), + commands, + ]; + } case "IdleElapsed": { const { id, generation, epoch } = event; const service = state.services.get(id); @@ -626,7 +863,7 @@ const applyEvent = ( ]; } case "ReadinessLost": { - const { id, generation } = event; + const { id, generation, cause } = event; const service = state.services.get(id); if ( service === undefined || @@ -634,17 +871,39 @@ const applyEvent = ( service.phase.generation !== generation ) break; - commands.push(LifecycleCommand.Reprobe({ id, generation })); + // Not terminal: the live session is kept, and the next waiter that needs it triggers a + // `Reprobe`. Waiters already awaiting this readiness get the check's failure. + const message = `${id} is not ready`; + const dependentsFailed = failExplicitDependentWaiters( + state, + id, + `prerequisite ${message}`, + cause, + commands, + ); return [ - setService(state, id, (s) => ({ - ...s, - phase: Phase.Running({ generation, ready: false }), - breaker: { - ...materializeStability(s.breaker, s.phase, now), - stableGeneration: undefined, - stableSince: undefined, - }, - })), + setService(dependentsFailed, id, (s) => { + const waiters = new Map(s.waiters); + for (const waiter of s.waiters.values()) { + if (!waiter.requireReady) continue; + waiters.delete(waiter.id); + commands.push( + LifecycleCommand.FailConnection({ id, waiterId: waiter.id, message, cause }), + ); + } + return { + ...s, + phase: Phase.Running({ generation, ready: false }), + breaker: { + ...materializeStability(s.breaker, s.phase, now), + stableGeneration: undefined, + stableSince: undefined, + }, + waiters, + readinessFailure: { cause, reprobed: s.reprobing }, + reprobing: false, + }; + }), commands, ]; } @@ -662,6 +921,8 @@ const applyEvent = ( ...s, phase: Phase.Running({ generation, ready: true }), breaker: { ...s.breaker, stableGeneration: generation, stableSince: now }, + readinessFailure: undefined, + reprobing: false, })), commands, ]; @@ -694,6 +955,7 @@ const applyEvent = ( const service = state.services.get(id); const spec = state.graph.services.get(id); if (service === undefined || spec === undefined) break; + if (rejectWhileDestroying(service, id, "start", commands)) break; // A service already `Running` has nothing left to force, and forcing it here would wrongly // survive into a later, unrelated idle cycle. One already `Stopping` is different: that // generation's end is imminent and this request's demand must carry through to the next one @@ -708,49 +970,41 @@ const applyEvent = ( breaker: initialBreaker, relaunchForced: s.relaunchForced || forced, })); - return [rearmPrerequisites(updated, id), commands]; + return [rearmPrerequisites(retryCleanup(updated, id, commands), id), commands]; } - case "StopRequested": { + case "ArmRequested": { const { id } = event; + const spec = state.graph.services.get(id); const service = state.services.get(id); - if (service === undefined) break; - // Checked independently of whether this service itself needs a `Stop` command: a `Failed` - // target with an open breaker must still protect a dependent's retained demand. - const blockers = activeDependents(state, id); - if (blockers.length > 0) { - commands.push( - LifecycleCommand.RequestRejected({ - id, - operation: "stop", - message: `${id} has active dependents: ${blockers.join(", ")}`, - }), - ); - break; - } - const winding = isUpOrWindingDown(service.phase) && service.phase._tag !== "Stopping"; - const failed = failAllWaiters( - service, - id, - `${id} was explicitly stopped`, - undefined, + if (spec === undefined || service === undefined || service.destroy !== undefined) break; + return [ + setService(state, id, (s) => ({ ...s, intent: spec.activation, breaker: initialBreaker })), commands, - ); - const generation = generationOf(failed.phase); - if (winding && generation !== undefined) - commands.push(LifecycleCommand.Stop({ id, generation })); - const phase = - winding && generation !== undefined - ? Phase.Stopping({ generation }) - : failed.phase._tag === "Failed" - ? Phase.Stopped() - : failed.phase; + ]; + } + case "StopRequested": { + const stopped = requestStop(state, event.id, commands); + return [stopped ?? state, commands]; + } + case "DestroyRequested": { + const stopped = requestStop(state, event.id, commands); + if (stopped === undefined) break; return [ - setService(state, id, () => ({ - ...failed, - intent: "stopped", - relaunchForced: false, - phase, - })), + setService(stopped, event.id, (s) => ({ ...s, destroy: s.destroy ?? "requested" })), + commands, + ]; + } + case "DestroyReleased": { + return [ + setService(state, event.id, (s) => + s.destroy === undefined + ? s + : { + ...s, + destroy: undefined, + storageReserved: s.destroy === "reserved" ? false : s.storageReserved, + }, + ), commands, ]; } @@ -759,6 +1013,7 @@ const applyEvent = ( const service = state.services.get(id); const spec = state.graph.services.get(id); if (service === undefined || spec === undefined) break; + if (rejectWhileDestroying(service, id, "restart", commands)) break; const blockers = activeDependents(state, id); if (blockers.length > 0) { commands.push( @@ -781,12 +1036,22 @@ const applyEvent = ( relaunchForced: true, phase: winding && generation !== undefined ? Phase.Stopping({ generation }) : s.phase, })); - return [rearmPrerequisites(updated, id), commands]; + return [rearmPrerequisites(retryCleanup(updated, id, commands), id), commands]; } case "StorageReserved": { const { id } = event; const service = state.services.get(id); if (service === undefined) break; + if (service.destroy !== undefined) { + commands.push( + LifecycleCommand.RequestRejected({ + id, + operation: "storage", + message: `${id} is being destroyed`, + }), + ); + break; + } if ( service.phase._tag !== "Stopped" || service.intent !== "stopped" || @@ -796,7 +1061,7 @@ const applyEvent = ( LifecycleCommand.RequestRejected({ id, operation: "storage", - message: `${id} must be stopped before a storage operation`, + message: `${id} must be stopped with wake disabled before modifying data`, }), ); break; @@ -804,7 +1069,13 @@ const applyEvent = ( return [setService(state, id, (s) => ({ ...s, storageReserved: true })), commands]; } case "StorageReleased": { - return [setService(state, event.id, (s) => ({ ...s, storageReserved: false })), commands]; + // Destroy's own reservation ends only with `DestroyReleased`. + return [ + setService(state, event.id, (s) => + s.destroy === "reserved" ? s : { ...s, storageReserved: false }, + ), + commands, + ]; } case "WaiterExpired": { const { id, waiterId } = event; @@ -871,7 +1142,7 @@ const applyEvent = ( const generationCounters = new Map(state.generationCounters); for (const spec of graph.services.values()) { const existing = state.services.get(spec.id); - services.set(spec.id, existing ?? initialService(spec, now)); + services.set(spec.id, existing ?? { ...initialService(spec, now), intent: "stopped" }); if (!generationCounters.has(spec.id)) generationCounters.set(spec.id, 1); } return [{ graph, services, generationCounters }, commands]; @@ -899,6 +1170,15 @@ const settle = ( let next = state; const emitted: Array = [...commands]; + // A pending destroy takes the storage reservation as soon as the service is stopped and free. + for (const [id, service] of next.services) + if ( + service.destroy === "requested" && + service.phase._tag === "Stopped" && + !service.storageReserved + ) + next = setService(next, id, (s) => ({ ...s, destroy: "reserved", storageReserved: true })); + for (const id of next.graph.order) { const spec = next.graph.services.get(id); const service = next.services.get(id); @@ -917,6 +1197,8 @@ const settle = ( ...s, phase: Phase.Starting({ generation, stage: "preparing" }), relaunchForced: false, + readinessFailure: undefined, + reprobing: false, })), generationCounters: new Map(next.generationCounters).set(id, generation + 1), }; @@ -925,16 +1207,34 @@ const settle = ( if (service.phase._tag === "Running") { const generation = service.phase.generation; + // A blocking unhealthy session is re-checked until it recovers, its generation ends or no + // waiter needs it; their budgets bound how long that lasts. + if ( + !service.phase.ready && + service.readinessFailure !== undefined && + !service.reprobing && + blocksWaiters(next, id) + ) { + emitted.push( + LifecycleCommand.Reprobe({ + id, + generation, + delayMillis: service.readinessFailure.reprobed ? reprobeSpacingMillis : 0, + }), + ); + next = setService(next, id, (s) => ({ ...s, reprobing: true })); + } if (prerequisitesSatisfied(next, id) && service.waiters.size > 0) { const ready = service.phase.ready; const admitted: Array = []; const expired: Array = []; for (const waiter of service.waiters.values()) { if (waiter.requireReady && !ready) continue; - if (now >= waiter.deadline) expired.push(waiter.id); + if (waiter.deadline !== undefined && now >= waiter.deadline) expired.push(waiter.id); else admitted.push(waiter.id); } if (admitted.length > 0 || expired.length > 0) { + const leased = admitted.filter((waiterId) => service.waiters.get(waiterId)?.lease).length; for (const waiterId of admitted) emitted.push(LifecycleCommand.AdmitConnection({ id, waiterId })); for (const waiterId of expired) @@ -950,7 +1250,12 @@ const settle = ( const waiters = new Map(s.waiters); for (const waiterId of admitted) waiters.delete(waiterId); for (const waiterId of expired) waiters.delete(waiterId); - return { ...s, leases: s.leases + admitted.length, waiters, lastActivity: now }; + return { + ...s, + leases: s.leases + leased, + waiters, + lastActivity: leased > 0 ? now : s.lastActivity, + }; }); } } @@ -996,13 +1301,14 @@ export const reduce = ( return settle(applied, now, directCommands); }; -/** A storage operation is admitted only while a service is stopped with no wake armed. */ +/** A storage operation is admitted only while a service is stopped, unarmed and not being destroyed. */ export const canRunStorage = (state: LifecycleState, id: string): boolean => { const service = state.services.get(id); return ( service !== undefined && service.intent === "stopped" && service.phase._tag === "Stopped" && - !service.storageReserved + !service.storageReserved && + service.destroy === undefined ); }; diff --git a/packages/stack/src/Lifecycle.unit.test.ts b/packages/stack/src/Lifecycle.unit.test.ts index e00d06f633..907f12ab3c 100644 --- a/packages/stack/src/Lifecycle.unit.test.ts +++ b/packages/stack/src/Lifecycle.unit.test.ts @@ -352,7 +352,8 @@ describe("Lifecycle reducer", () => { expect(state.services.get("c")?.phase).toEqual({ _tag: "Running", generation: 1, ready: true }); const blocked = run(state, [{ event: open("c", 2), now: 50 }]); - expect(tagsOf(blocked.commandsByStep[0])).toEqual(["ArmWaiterTimeout"]); + expect(tagsOf(blocked.commandsByStep[0])).toEqual(["ArmWaiterTimeout", "Reprobe"]); + expect(commandTagged(at(blocked.commandsByStep, 0), "Reprobe").id).toBe("a"); expect(blocked.state.services.get("c")?.waiters.size).toBe(1); const expired = run(blocked.state, [ @@ -763,7 +764,7 @@ describe("Lifecycle reducer", () => { }); const readyRequiring = run(state, [{ event: open("functions", 2), now: 20 }]); - expect(tagsOf(readyRequiring.commandsByStep[0])).toEqual(["ArmWaiterTimeout"]); + expect(tagsOf(readyRequiring.commandsByStep[0])).toEqual(["ArmWaiterTimeout", "Reprobe"]); const inspector = run(state, [ { @@ -778,7 +779,7 @@ describe("Lifecycle reducer", () => { expect(tagsOf(inspector.commandsByStep[0])).toEqual(["AdmitConnection"]); }); - it("treats a readiness loss as non-terminal with a reprobe, unlike an exit which counts toward the breaker", () => { + it("treats a readiness loss as non-terminal, re-probing once for the next waiter, unlike an exit which counts toward the breaker", () => { const graph = makeGraph([lazy("api")]); const { state: ready } = run(initialState(graph, 0), [ { event: open("api", 1), now: 0 }, @@ -791,7 +792,7 @@ describe("Lifecycle reducer", () => { now: 20, }, ]); - expect(tagsOf(lostCommands[0])).toEqual(["Reprobe"]); + expect(lostCommands[0]).toEqual([]); expect(afterReadinessLoss.services.get("api")?.phase).toEqual({ _tag: "Running", generation: 1, @@ -799,9 +800,17 @@ describe("Lifecycle reducer", () => { }); expect(afterReadinessLoss.services.get("api")?.breaker.consecutiveFailures).toBe(0); - const { state: recovered } = run(afterReadinessLoss, [ + const { state: probing, commandsByStep: probeCommands } = run(afterReadinessLoss, [ + { event: open("api", 2), now: 21 }, + { event: open("api", 3), now: 22 }, + ]); + expect(tagsOf(probeCommands[0])).toEqual(["ArmWaiterTimeout", "Reprobe"]); + expect(tagsOf(probeCommands[1])).toEqual(["ArmWaiterTimeout"]); + + const { state: recovered, commandsByStep: recoveredCommands } = run(probing, [ { event: LifecycleEvent.ReadinessRecovered({ id: "api", generation: 1 }), now: 30 }, ]); + expect(tagsOf(recoveredCommands[0])).toEqual(["AdmitConnection", "AdmitConnection"]); expect(recovered.services.get("api")?.phase).toEqual({ _tag: "Running", generation: 1, @@ -909,6 +918,91 @@ describe("Lifecycle reducer", () => { ]); expect(tagsOf(released.commandsByStep[0])).toEqual(["Launch"]); }); + + it("hands a running storage operation's reservation to a destroy requested meanwhile", () => { + const graph = makeGraph([lazy("database")]); + let state = initialState(graph, 0); + ({ state } = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "database" }), now: 0 }, + { event: LifecycleEvent.StorageReserved({ id: "database" }), now: 1 }, + { event: LifecycleEvent.DestroyRequested({ id: "database" }), now: 2 }, + ])); + expect(state.services.get("database")?.destroy).toBe("requested"); + + const refused = run(state, [ + { event: LifecycleEvent.StorageReserved({ id: "database" }), now: 3 }, + { event: LifecycleEvent.StartRequested({ id: "database" }), now: 3 }, + ]); + expect(refused.commandsByStep.map(tagsOf)).toEqual([ + ["RequestRejected"], + ["RequestRejected"], + ]); + expect(canRunStorage(refused.state, "database")).toBe(false); + + ({ state } = run(refused.state, [ + { event: LifecycleEvent.StorageReleased({ id: "database" }), now: 4 }, + ])); + expect(state.services.get("database")).toMatchObject({ + destroy: "reserved", + storageReserved: true, + }); + }); + + it("reserves storage for a destroy once its running service has stopped", () => { + const graph = makeGraph([lazy("database")]); + let state = initialState(graph, 0); + ({ state } = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "database" }), now: 0 }, + ])); + const generation = startingGeneration(state, "database"); + + const requested = run(state, [ + { event: LifecycleEvent.DestroyRequested({ id: "database" }), now: 1 }, + ]); + expect(tagsOf(requested.commandsByStep[0])).toEqual(["Stop"]); + expect(requested.state.services.get("database")?.destroy).toBe("requested"); + + ({ state } = run(requested.state, [ + { + event: LifecycleEvent.Exited({ + id: "database", + generation, + cause: undefined, + requested: true, + }), + now: 2, + }, + ])); + expect(state.services.get("database")).toMatchObject({ + phase: { _tag: "Stopped" }, + destroy: "reserved", + storageReserved: true, + }); + }); + + it("makes a service usable again once a destroy releases it", () => { + const graph = makeGraph([lazy("database")]); + let state = initialState(graph, 0); + ({ state } = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "database" }), now: 0 }, + { event: LifecycleEvent.DestroyRequested({ id: "database" }), now: 1 }, + ])); + expect(state.services.get("database")?.destroy).toBe("reserved"); + // A storage operation's release can't end the destroy's own reservation. + ({ state } = run(state, [ + { event: LifecycleEvent.StorageReleased({ id: "database" }), now: 2 }, + ])); + expect(canRunStorage(state, "database")).toBe(false); + + ({ state } = run(state, [ + { event: LifecycleEvent.DestroyReleased({ id: "database" }), now: 3 }, + ])); + expect(state.services.get("database")).toMatchObject({ + destroy: undefined, + storageReserved: false, + }); + expect(canRunStorage(state, "database")).toBe(true); + }); }); it("rejects new admission for an explicitly stopped service without creating a waiter", () => { @@ -1291,4 +1385,338 @@ describe("Lifecycle reducer", () => { ])); expect(state.services.get("api")?.idleArmedEpoch).toBe(2); }); + + it("fails only the waiters awaiting a failed readiness check, keeping the session and inspector waiters", () => { + const graph = makeGraph([lazy("database"), lazy("api", { prerequisites: ["database"] })]); + let state = initialState(graph, 0); + ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "database", generation: 1 }), now: 1 }, + { event: LifecycleEvent.SessionAvailable({ id: "api", generation: 1 }), now: 2 }, + { event: open("api", 2, false), now: 3 }, + ])); + + const lost = run(state, [ + { event: LifecycleEvent.ReadinessLost({ id: "api", generation: 1, cause: "503" }), now: 4 }, + ]); + const failed = commandTagged(at(lost.commandsByStep, 0), "FailConnection"); + expect(failed).toMatchObject({ waiterId: 1, message: "api is not ready", cause: "503" }); + expect(tagsOf(lost.commandsByStep[0])).toEqual(["FailConnection"]); + expect(lost.state.services.get("api")?.phase).toEqual({ + _tag: "Running", + generation: 1, + ready: false, + }); + expect(lost.state.services.get("api")?.leases).toBe(1); + }); + + it("gives an explicit readiness wait no budget and no lease, while it still wakes the service", () => { + let state = initialState(makeGraph([lazy("api")]), 0); + let commandsByStep: ReadonlyArray>; + ({ state, commandsByStep } = run(state, [ + { + event: LifecycleEvent.ReadinessAwaited({ id: "api", waiterId: 1, requireReady: true }), + now: 0, + }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["Launch"]); + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 500_000 }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["AdmitConnection", "ArmIdleTimer"]); + expect(state.services.get("api")?.leases).toBe(0); + }); + + it("adds services unarmed through a graph update until an arm or start request", () => { + let state = initialState(makeGraph([]), 0); + let commandsByStep: ReadonlyArray>; + const graph = makeGraph([ + { id: "database", activation: "eager", prerequisites: [] }, + lazy("api", { prerequisites: ["database"] }), + ]); + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.GraphUpdated({ graph }), now: 0 }, + ])); + expect(commandsByStep[0]).toEqual([]); + expect(state.services.get("database")?.intent).toBe("stopped"); + expect(state.services.get("api")?.intent).toBe("stopped"); + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.ArmRequested({ id: "api" }), now: 1 }, + ])); + expect(commandsByStep[0]).toEqual([]); + expect(state.services.get("api")?.intent).toBe("lazy"); + expect(state.services.get("database")?.intent).toBe("stopped"); + + ({ commandsByStep } = run(state, [ + { event: LifecycleEvent.ArmRequested({ id: "database" }), now: 2 }, + ])); + expect(commandTagged(at(commandsByStep, 0), "Launch").id).toBe("database"); + }); + + it("lets an eager service's standing demand lapse once it fails, until traffic re-wakes it", () => { + const graph = makeGraph([{ id: "database", activation: "eager", prerequisites: [] }]); + let state = initialState(graph, 0); + let commandsByStep: ReadonlyArray>; + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "database" }), now: 0 }, + { + event: LifecycleEvent.LaunchFailed({ id: "database", generation: 1, cause: "boom" }), + now: 1, + }, + ])); + expect(commandsByStep[1]).toEqual([]); + expect(state.services.get("database")?.phase._tag).toBe("Failed"); + + ({ commandsByStep } = run(state, [{ event: open("database", 1), now: 2 }])); + expect(commandTagged(at(commandsByStep, 0), "Launch").generation).toBe(2); + }); + + it("ends an explicit wait on a dependent with its prerequisite's failure, while traffic keeps waiting", () => { + const graph = makeGraph([lazy("database"), lazy("api", { prerequisites: ["database"] })]); + let state = initialState(graph, 0); + ({ state } = run(state, [ + { event: open("api", 1), now: 0 }, + { + event: LifecycleEvent.ReadinessAwaited({ id: "api", waiterId: 2, requireReady: false }), + now: 0, + }, + ])); + + const failed = run(state, [ + { + event: LifecycleEvent.LaunchFailed({ id: "database", generation: 1, cause: "boom" }), + now: 1, + }, + ]); + const failure = commandTagged(at(failed.commandsByStep, 0), "FailConnection"); + expect(failure).toMatchObject({ + id: "api", + waiterId: 2, + message: "prerequisite database failed", + }); + expect([...(failed.state.services.get("api")?.waiters.keys() ?? [])]).toEqual([1]); + }); + + describe("re-checking a blocking session", () => { + const blocked = () => { + const graph = makeGraph([ + lazy("analytics"), + lazy("studio", { prerequisites: ["analytics"] }), + ]); + let state = initialState(graph, 0); + ({ state } = run(state, [ + { event: open("studio", 1), now: 0 }, + { event: LifecycleEvent.SessionAvailable({ id: "analytics", generation: 1 }), now: 1 }, + ])); + return state; + }; + const lost = (now: number) => ({ + event: LifecycleEvent.ReadinessLost({ id: "analytics", generation: 1, cause: "503" }), + now, + }); + + it("keeps re-probing one at a time, spaced after a failed reprobe, until the dependent is served", () => { + const { state, commandsByStep } = run(blocked(), [ + lost(2), + { event: open("studio", 2), now: 3 }, + lost(4), + lost(1_005), + { + event: LifecycleEvent.ReadinessRecovered({ id: "analytics", generation: 1 }), + now: 2_006, + }, + ]); + expect(commandTagged(at(commandsByStep, 0), "Reprobe")).toMatchObject({ + id: "analytics", + generation: 1, + delayMillis: 0, + }); + expect(tagsOf(commandsByStep[1])).toEqual(["ArmWaiterTimeout"]); + expect(commandTagged(at(commandsByStep, 2), "Reprobe").delayMillis).toBe(1_000); + expect(commandTagged(at(commandsByStep, 3), "Reprobe").delayMillis).toBe(1_000); + expect(commandTagged(at(commandsByStep, 4), "Launch").id).toBe("studio"); + expect(state.services.get("analytics")?.reprobing).toBe(false); + }); + + it("stops re-probing once no waiter is blocked or the generation ends", () => { + let state = blocked(); + let commandsByStep: ReadonlyArray>; + ({ state } = run(state, [lost(2)])); + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.WaiterExpired({ id: "studio", waiterId: 1 }), now: 3 }, + lost(4), + ])); + expect(tagsOf(commandsByStep[1])).toEqual([]); + + ({ state } = run(state, [{ event: open("studio", 2), now: 5 }])); + ({ commandsByStep } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "analytics", + generation: 1, + cause: "gone", + requested: false, + }), + now: 6, + }, + lost(7), + ])); + expect(tagsOf(commandsByStep[1]).includes("Reprobe")).toBe(false); + }); + }); + + describe("failed cleanup", () => { + const runningApi = () => { + let state = initialState(makeGraph([lazy("api")]), 0); + ({ state } = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "api" }), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 1 }, + ])); + return state; + }; + + it("holds the generation in cleanup until a retried stop confirms it, refusing storage and launches", () => { + let state = runningApi(); + let commandsByStep: ReadonlyArray>; + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "api" }), now: 2 }, + { + event: LifecycleEvent.StopFailed({ id: "api", generation: 1, cause: "busy" }), + now: 3, + }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["Stop"]); + expect(commandsByStep[1]).toEqual([]); + expect(state.services.get("api")?.phase).toEqual({ _tag: "Stopping", generation: 1 }); + expect(canRunStorage(state, "api")).toBe(false); + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.StorageReserved({ id: "api" }), now: 4 }, + { event: LifecycleEvent.StopRequested({ id: "api" }), now: 5 }, + { event: LifecycleEvent.StopRequested({ id: "api" }), now: 6 }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["RequestRejected"]); + expect(commandTagged(at(commandsByStep, 1), "Stop").generation).toBe(1); + expect(commandsByStep[2]).toEqual([]); + + ({ state } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "api", + generation: 1, + cause: undefined, + requested: true, + }), + now: 7, + }, + ])); + expect(canRunStorage(state, "api")).toBe(true); + }); + + it("counts a crash whose cleanup failed once, not again when the retried cleanup confirms it", () => { + let state = runningApi(); + ({ state } = run(state, [ + { + event: LifecycleEvent.StopFailed({ + id: "api", + generation: 1, + cause: "busy", + failure: "segfault", + }), + now: 2, + }, + { event: LifecycleEvent.StopRequested({ id: "api" }), now: 3 }, + { + event: LifecycleEvent.StopFailed({ id: "api", generation: 1, cause: "busy" }), + now: 4, + }, + { event: LifecycleEvent.StopRequested({ id: "api" }), now: 5 }, + { + event: LifecycleEvent.Exited({ + id: "api", + generation: 1, + cause: undefined, + requested: true, + }), + now: 6, + }, + ])); + expect(state.services.get("api")?.breaker).toMatchObject({ + consecutiveFailures: 1, + lastCause: "segfault", + }); + }); + + it("fails a restart's waiters when the stop fails, and retries the cleanup for the next waiter", () => { + let state = runningApi(); + let commandsByStep: ReadonlyArray>; + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.RestartRequested({ id: "api" }), now: 2 }, + { + event: LifecycleEvent.ReadinessAwaited({ id: "api", waiterId: 1, requireReady: false }), + now: 2, + }, + { + event: LifecycleEvent.StopFailed({ id: "api", generation: 1, cause: "busy" }), + now: 3, + }, + ])); + expect(commandTagged(at(commandsByStep, 2), "FailConnection")).toMatchObject({ + waiterId: 1, + message: "api cleanup failed", + cause: "busy", + }); + expect(state.services.get("api")?.relaunchForced).toBe(false); + + ({ commandsByStep } = run(state, [{ event: open("api", 2), now: 4 }])); + expect(tagsOf(commandsByStep[0])).toEqual(["ArmWaiterTimeout", "Stop"]); + }); + }); + + it("caps only traffic waiters and dates a waiter's budget from the client's arrival", () => { + let state = initialState(makeGraph([lazy("api", { waiterBudgetMillis: 100 })]), 0); + let commandsByStep: ReadonlyArray>; + ({ state } = run( + state, + Array.from({ length: 256 }, (_, index) => ({ event: open("api", index), now: 0 })), + )); + ({ state, commandsByStep } = run(state, [ + { event: open("api", 1_000), now: 0 }, + { + event: LifecycleEvent.ReadinessAwaited({ id: "api", waiterId: 1_001, requireReady: true }), + now: 0, + }, + { + event: LifecycleEvent.ConnectionOpened({ + id: "api", + waiterId: 1_002, + requireReady: true, + openedAt: 0, + }), + now: 40, + }, + ])); + expect(commandTagged(at(commandsByStep, 0), "FailConnection").message).toBe( + "api has too many waiters", + ); + expect(commandsByStep[1]).toEqual([]); + expect(state.services.get("api")?.waiters.has(1_001)).toBe(true); + expect(tagsOf(commandsByStep[2])).toEqual(["FailConnection"]); + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.WaiterCancelled({ id: "api", waiterId: 0 }), now: 50 }, + { + event: LifecycleEvent.ConnectionOpened({ + id: "api", + waiterId: 1_003, + requireReady: true, + openedAt: 20, + }), + now: 50, + }, + ])); + expect(commandTagged(at(commandsByStep, 1), "ArmWaiterTimeout").deadline).toBe(120); + }); }); diff --git a/packages/stack/src/Orchestrator.integration.test.ts b/packages/stack/src/Orchestrator.integration.test.ts index 1fe51757ff..e99cdbfe63 100644 --- a/packages/stack/src/Orchestrator.integration.test.ts +++ b/packages/stack/src/Orchestrator.integration.test.ts @@ -1,10 +1,32 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; -import { Cause, Deferred, Effect, Exit, Fiber, Option, Ref, Schema, Scope, Stream } from "effect"; +import { + Cause, + Context, + Deferred, + Duration, + Effect, + Exit, + Fiber, + FileSystem, + Layer, + Option, + Queue, + Ref, + Schema, + Scope, + Stream, +} from "effect"; import * as TestClock from "effect/testing/TestClock"; +import { HttpClient } from "effect/unstable/http"; +import { createServer } from "node:http"; // oxlint-disable-line effecttsgo/node-builtin-import -- real backend behind the listener. import { captureLogs } from "../tests/logs.ts"; +import * as Network from "./Network.ts"; import * as Orchestrator from "./Orchestrator.ts"; import type { RegisteredInstance } from "./Orchestrator.ts"; +import { ProxyError } from "./Proxy.ts"; import { makeService, ServiceError } from "./Service.ts"; +import * as StackNamespace from "./StackNamespace.ts"; const failure = (message: string) => new ServiceError({ operation: "fixture", message }); const makeTestOrchestrator = () => Orchestrator.make(); @@ -20,7 +42,10 @@ const makeInstance = ( readonly launch?: Effect.Effect; readonly stop?: Effect.Effect; readonly removeData?: Effect.Effect; + /** Resolves the first launch's runtime exit; later launches exit only when stopped. */ readonly exit?: Deferred.Deferred>; + /** Every launch's runtime exit, for workloads that crash on demand. */ + readonly crash?: Effect.Effect>; readonly events?: Ref.Ref>; } = {}, ) => @@ -37,13 +62,18 @@ const makeInstance = ( launch: ({ config }) => Effect.gen(function* () { yield* options.launch ?? Effect.void; - yield* Ref.update(starts, (values) => [...values, config]); + const launched = yield* Ref.updateAndGet(starts, (values) => [...values, config]); yield* event("start"); const exited = yield* Deferred.make>(); + const exit = + options.crash ?? + (options.exit !== undefined && launched.length === 1 + ? Deferred.await(options.exit) + : Deferred.await(exited)); return { health: options.health ?? Effect.void, ...(options.probe === undefined ? {} : { probe: options.probe }), - exit: Deferred.await(options.exit ?? exited), + exit, stop: (options.stop ?? Effect.void).pipe( Effect.andThen(event("stop")), Effect.andThen(Deferred.succeed(exited, Exit.void)), @@ -54,41 +84,79 @@ const makeInstance = ( }), removeData: () => event("destroy").pipe(Effect.andThen(options.removeData ?? Effect.void)), }, - { id, config: {}, coordinate: orchestrator.admissionFor(id) }, + { id, config: {}, report: orchestrator.report }, ); + const configFor = (inputs: Record, candidate: unknown) => + candidate === undefined + ? Effect.succeed(inputs) + : Schema.decodeUnknownEffect(Schema.Record(Schema.String, Schema.String))(candidate).pipe( + Effect.mapError(() => failure("Invalid configuration")), + ); const instance: RegisteredInstance = { id, service: id, core, - startAt: (revision, inputs, wake, guard) => core.startAt(revision, inputs, wake, guard), - restart: (revision, inputs, candidate, guard) => - (candidate === undefined - ? Effect.succeed(inputs) - : Schema.decodeUnknownEffect(Schema.Record(Schema.String, Schema.String))(candidate).pipe( - Effect.mapError(() => failure("Invalid configuration")), - ) - ).pipe(Effect.flatMap((config) => core.restart(config, revision, guard))), + launch: (generation, inputs, candidate) => + configFor(inputs, candidate).pipe( + Effect.flatMap((config) => core.launch(generation, config)), + ), + prepare: (inputs, candidate) => + configFor(inputs, candidate).pipe(Effect.flatMap((config) => core.prepare(config))), bind: options.bind ?? Ref.set(bound, true), - close: core.get.pipe( + close: orchestrator.status(id).pipe( Effect.flatMap((state) => state.lifecycle === "stopped" && !state.wakeEnabled ? Ref.set(bound, false) : Effect.void, ), + Effect.mapError((error) => failure(error.message)), ), hasEndpoint: options.endpoint ?? true, inputs: ["databaseUrl"], outputs: { url: Effect.succeed(`postgres://${id}`) }, }; yield* orchestrator.register(instance); - return { ...instance, starts, bound }; + return { + ...instance, + starts, + bound, + status: orchestrator.status(id), + ready: orchestrator.ready(id), + observation: orchestrator.changes(id), + }; }); -const stopped = (instance: RegisteredInstance) => - instance.core.observation.pipe( +type Instance = Effect.Success>; + +const stopped = (instance: Instance) => + instance.observation.pipe( Stream.filter((state) => state.lifecycle === "stopped" && state.currentOperation === undefined), Stream.take(1), Stream.runDrain, ); +/** + * Forks a wait for the next lifecycle transition after now. It returns once the subscription has + * seen the current state, so a transition triggered afterwards can't be missed. + */ +const nextTransition = (observation: Stream.Stream) => + Effect.gen(function* () { + const subscribed = yield* Deferred.make(); + const transition = yield* observation.pipe( + Stream.zipWithIndex, + Stream.tap(([, index]) => + index === 0 ? Deferred.succeed(subscribed, undefined) : Effect.void, + ), + Stream.drop(1), + Stream.take(1), + Stream.runDrain, + Effect.forkChild, + ); + yield* Deferred.await(subscribed); + return transition; + }); + +const errorOf = (exit: Exit.Exit) => + Exit.isFailure(exit) ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) : undefined; + describe("service composition", () => { it.live("starts independent eager services concurrently", () => Effect.scoped( @@ -193,11 +261,11 @@ describe("service composition", () => { yield* Deferred.await(databaseLaunched); yield* Deferred.await(authLaunched); yield* Deferred.succeed(authHealthy, undefined); - yield* auth.core.ready; + yield* auth.ready; yield* Deferred.succeed(databaseHealthGate, undefined); - yield* database.core.ready; + yield* database.ready; yield* Deferred.await(restLaunched); - yield* rest.core.ready; + yield* rest.ready; return { boundInputs: yield* Ref.get(rest.starts), eventOrder: yield* Ref.get(events), @@ -247,13 +315,7 @@ describe("service composition", () => { dependencies: [{ from: "prerequisite", to: "descendant" }], }); - const result = yield* orchestrator.startComposition.pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isSuccess(result)) - return yield* Effect.die( - "composition with an unhealthy prerequisite unexpectedly passed", - ); - const error = Option.getOrUndefined(Cause.findErrorOption(result.cause)); + const error = errorOf(yield* orchestrator.startComposition.pipe(Effect.exit)); expect(error).toBeInstanceOf(Orchestrator.OrchestratorError); if (!(error instanceof Orchestrator.OrchestratorError)) return yield* Effect.die("composition failure did not include ordered outcomes"); @@ -265,12 +327,7 @@ describe("service composition", () => { ["independent", true], ]); const descendantOutcome = error.outcomes?.find(({ id }) => id === "descendant")?.result; - if (descendantOutcome === undefined || Exit.isSuccess(descendantOutcome)) - return yield* Effect.die("unhealthy prerequisite did not block descendant startup"); - const descendantError = Option.getOrUndefined( - Cause.findErrorOption(descendantOutcome.cause), - ); - expect(descendantError).toMatchObject({ + expect(descendantOutcome && errorOf(descendantOutcome)).toMatchObject({ message: "Blocked by prerequisites: prerequisite", }); expect(yield* Ref.get(descendant.starts)).toEqual([]); @@ -306,11 +363,7 @@ describe("service composition", () => { ], }); - const result = yield* orchestrator.startComposition.pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isSuccess(result)) - return yield* Effect.die("composition with a prebinding failure unexpectedly passed"); - const error = Option.getOrUndefined(Cause.findErrorOption(result.cause)); + const error = errorOf(yield* orchestrator.startComposition.pipe(Effect.exit)); if (!(error instanceof Orchestrator.OrchestratorError)) return yield* Effect.die("prebinding failure did not return composition outcomes"); expect( @@ -322,17 +375,15 @@ describe("service composition", () => { ["independent", true], ]); const blocked = error.outcomes?.find(({ id }) => id === "descendant")?.result; - if (blocked === undefined || Exit.isSuccess(blocked)) - return yield* Effect.die("eager descendant unexpectedly started"); - expect(Option.getOrUndefined(Cause.findErrorOption(blocked.cause))).toMatchObject({ + expect(blocked && errorOf(blocked)).toMatchObject({ message: "Blocked by prerequisites: prerequisite", }); expect(yield* Ref.get(descendant.starts)).toEqual([]); expect(yield* Ref.get(lazyDescendant.starts)).toEqual([]); - expect((yield* lazyDescendant.core.get).wakeEnabled).toBe(true); + expect((yield* lazyDescendant.status).wakeEnabled).toBe(true); expect(yield* Ref.get(independent.starts)).toHaveLength(1); yield* orchestrator.stopNamespace; - expect((yield* lazyDescendant.core.get).wakeEnabled).toBe(false); + expect((yield* lazyDescendant.status).wakeEnabled).toBe(false); }), ), ); @@ -349,7 +400,7 @@ describe("service composition", () => { }); const dependent = yield* makeInstance(orchestrator, "dependent"); const independentLazy = yield* makeInstance(orchestrator, "independent-lazy"); - const independentArmed = independentLazy.core.observation.pipe( + const independentArmed = independentLazy.observation.pipe( Stream.filter((state) => state.wakeEnabled), Stream.take(1), Stream.runDrain, @@ -372,14 +423,14 @@ describe("service composition", () => { const observedIndependentArm = yield* Fiber.join(armedObserver).pipe( Effect.timeoutOption("5 seconds"), ); - expect((yield* prerequisite.core.get).health).toBe("starting"); - expect((yield* dependent.core.get).wakeEnabled).toBe(false); - expect((yield* independentLazy.core.get).wakeEnabled).toBe(true); + expect((yield* prerequisite.status).health).toBe("starting"); + expect((yield* dependent.status).wakeEnabled).toBe(false); + expect((yield* independentLazy.status).wakeEnabled).toBe(true); yield* Deferred.succeed(healthGate, undefined); expect(Option.isSome(observedIndependentArm)).toBe(true); yield* Fiber.join(composition); - expect((yield* dependent.core.get).wakeEnabled).toBe(true); + expect((yield* dependent.status).wakeEnabled).toBe(true); expect(yield* Ref.get(dependent.starts)).toEqual([]); yield* orchestrator.stopNamespace; }), @@ -399,9 +450,7 @@ describe("service composition", () => { Effect.andThen(Deferred.await(launchGate)), ), }); - const descendant = yield* makeInstance(orchestrator, "descendant", { - events, - }); + const descendant = yield* makeInstance(orchestrator, "descendant", { events }); yield* Effect.addFinalizer(() => Deferred.succeed(launchGate, undefined).pipe(Effect.asVoid), ); @@ -420,12 +469,11 @@ describe("service composition", () => { ); yield* Deferred.succeed(launchGate, undefined); expect(Option.isSome(interruptFinished)).toBe(true); - const interrupted = yield* Fiber.await(composition); - expect(Exit.isFailure(interrupted)).toBe(true); + expect(Exit.isFailure(yield* Fiber.await(composition))).toBe(true); expect(yield* Ref.get(descendant.starts)).toEqual([]); - yield* prerequisite.core.ready; - expect((yield* prerequisite.core.get).lifecycle).toBe("running"); + yield* prerequisite.ready; + expect((yield* prerequisite.status).lifecycle).toBe("running"); expect(yield* Ref.get(descendant.starts)).toEqual([]); yield* orchestrator.stopNamespace; expect( @@ -447,18 +495,18 @@ describe("service composition", () => { health: Deferred.await(health), }); yield* orchestrator.start("functions"); - expect((yield* functions.core.get).health).toBe("starting"); + expect((yield* functions.status).health).toBe("starting"); yield* Effect.scoped(orchestrator.acquire("functions", false)); - expect((yield* functions.core.get).health).toBe("starting"); + expect((yield* functions.status).health).toBe("starting"); yield* Deferred.succeed(health, undefined); yield* Effect.scoped(orchestrator.acquire("functions")); - expect((yield* functions.core.get).health).toBe("healthy"); + expect((yield* functions.status).health).toBe("healthy"); }), ), ); it.live( - "launches an individual service before health and invalidates a dependent readiness wait on stop", + "rejects stopping a prerequisite while a dependent's start waits on it, then starts the dependent", () => Effect.scoped( Effect.gen(function* () { @@ -475,45 +523,19 @@ describe("service composition", () => { ], }); yield* orchestrator.start("database"); - expect((yield* database.core.get).health).toBe("starting"); + expect((yield* database.status).health).toBe("starting"); const dependent = yield* orchestrator .start("rest") .pipe(Effect.forkChild({ startImmediately: true })); - yield* orchestrator.stop("database"); - expect(Exit.isFailure(yield* Fiber.await(dependent))).toBe(true); - expect(yield* Ref.get(rest.starts)).toEqual([]); - expect((yield* database.core.get).lifecycle).toBe("stopped"); - }), - ), - ); - it.live( - "rejects a stale dependency plan after its prerequisite was stopped and replaced during preparation", - () => - Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - const preparing = yield* Deferred.make(); - const continuePreparation = yield* Deferred.make(); - const database = yield* makeInstance(orchestrator, "database"); - const rest = yield* makeInstance(orchestrator, "rest", { - prepare: Deferred.succeed(preparing, undefined).pipe( - Effect.andThen(Deferred.await(continuePreparation)), - ), - }); - yield* orchestrator.configure({ - members: [{ id: "rest", activation: "eager" }], - dependencies: [{ from: "database", to: "rest" }], - }); - const request = yield* orchestrator.start("rest").pipe(Effect.forkChild); - yield* Deferred.await(preparing); - yield* orchestrator.stop("database"); - yield* orchestrator.start("database"); - yield* database.core.ready; - yield* Deferred.succeed(continuePreparation, undefined); - expect(Exit.isFailure(yield* Fiber.await(request))).toBe(true); + const rejected = yield* orchestrator.stop("database").pipe(Effect.flip); + expect(rejected.message).toBe("database has active dependents: rest"); expect(yield* Ref.get(rest.starts)).toEqual([]); - expect((yield* database.core.get).lifecycle).toBe("running"); + + yield* Deferred.succeed(health, undefined); + yield* Fiber.join(dependent); + expect(yield* Ref.get(rest.starts)).toEqual([{ databaseUrl: "postgres://database" }]); + yield* orchestrator.stopNamespace; }), ), ); @@ -551,8 +573,8 @@ describe("service composition", () => { ]); expect(yield* Ref.get(standalone.starts)).toHaveLength(1); yield* orchestrator.stopNamespace; - expect((yield* standalone.core.get).lifecycle).toBe("stopped"); - expect((yield* database.core.get).lifecycle).toBe("stopped"); + expect((yield* standalone.status).lifecycle).toBe("stopped"); + expect((yield* database.status).lifecycle).toBe("stopped"); }), ), ); @@ -580,18 +602,19 @@ describe("service composition", () => { yield* orchestrator.startComposition; const result = yield* orchestrator.restartComposition.pipe(Effect.exit); expect(Exit.isFailure(result)).toBe(true); - expect((yield* bad.core.get).lifecycle).toBe("stopping"); - expect((yield* independent.core.get).lifecycle).toBe("stopped"); + expect((yield* bad.status).lifecycle).toBe("stopping"); + expect((yield* independent.status).lifecycle).toBe("stopped"); expect(yield* Ref.get(bad.starts)).toHaveLength(1); expect(yield* Ref.get(independent.starts)).toHaveLength(1); yield* Ref.set(failStop, false); yield* orchestrator.stopNamespace; + expect((yield* bad.status).lifecycle).toBe("stopped"); }), ), ); it.live( - "binds lazy endpoints without launching, counts traffic, and sleeps idle prerequisites after dependents", + "binds lazy endpoints without launching, counts traffic, and sleeps an idle prerequisite after its dependent", () => Effect.scoped( Effect.gen(function* () { @@ -611,17 +634,19 @@ describe("service composition", () => { expect(yield* Ref.get(database.starts)).toEqual([]); const requestScope = yield* Scope.make(); yield* orchestrator.acquire("rest").pipe(Scope.provide(requestScope)); - expect((yield* rest.core.get).health).toBe("healthy"); + expect((yield* rest.status).health).toBe("healthy"); yield* TestClock.adjust("2 seconds"); - expect((yield* rest.core.get).lifecycle).toBe("running"); - expect((yield* database.core.get).lifecycle).toBe("running"); + expect((yield* rest.status).lifecycle).toBe("running"); + expect((yield* database.status).lifecycle).toBe("running"); yield* Scope.close(requestScope, Exit.void); const restStopped = yield* stopped(rest).pipe(Effect.forkChild); const databaseStopped = yield* stopped(database).pipe(Effect.forkChild); yield* TestClock.adjust("1 second"); yield* Fiber.join(restStopped); + expect((yield* database.status).lifecycle).toBe("running"); + yield* TestClock.adjust("100 millis"); yield* Fiber.join(databaseStopped); - expect((yield* database.core.get).wakeEnabled).toBe(true); + expect((yield* database.status).wakeEnabled).toBe(true); expect(yield* Ref.get(rest.bound)).toBe(true); yield* orchestrator.stopComposition; yield* orchestrator.acquire("rest").pipe(Effect.flip); @@ -655,7 +680,7 @@ describe("service composition", () => { yield* orchestrator.acquire("rest").pipe(Scope.provide(requestScope)); yield* Scope.close(requestScope, Exit.void); yield* TestClock.adjust("200 millis"); - expect((yield* database.core.get).lifecycle).toBe("running"); + expect((yield* database.status).lifecycle).toBe("running"); const restStopped = yield* stopped(rest).pipe(Effect.forkChild); const databaseStopped = yield* stopped(database).pipe(Effect.forkChild); yield* Deferred.succeed( @@ -663,12 +688,12 @@ describe("service composition", () => { Exit.fail(new ServiceError({ operation: "process", message: "crashed" })), ); yield* Deferred.await(cleanupStarted); - expect((yield* database.core.get).lifecycle).toBe("running"); + expect((yield* database.status).lifecycle).toBe("running"); yield* Deferred.succeed(cleanupGate, undefined); yield* Fiber.join(restStopped); yield* TestClock.adjust("1 second"); yield* Fiber.join(databaseStopped); - expect((yield* database.core.get).lifecycle).toBe("stopped"); + expect((yield* database.status).lifecycle).toBe("stopped"); yield* orchestrator.stopNamespace; }), ).pipe(Effect.provide(TestClock.layer())), @@ -693,10 +718,7 @@ describe("service composition", () => { yield* orchestrator.start("next"); const result = yield* Effect.exit(orchestrator.destroyNamespace); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isSuccess(result)) - return yield* Effect.die("namespace destroy unexpectedly passed"); - const error = Option.getOrUndefined(Cause.findErrorOption(result.cause)); + const error = errorOf(result); expect(error).toBeInstanceOf(Orchestrator.OrchestratorError); if (!(error instanceof Orchestrator.OrchestratorError)) return yield* Effect.die("namespace destroy did not return a combined error"); @@ -722,220 +744,1142 @@ describe("service composition", () => { ); }); -it.live("joins an in-progress wake and wakes again after an admitted idle stop", () => - Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - const launching = yield* Deferred.make(); - const launch = yield* Deferred.make(); - const stopping = yield* Deferred.make(); - const stop = yield* Deferred.make(); - const instance = yield* makeInstance(orchestrator, "api", { - launch: Deferred.succeed(launching, undefined).pipe(Effect.andThen(Deferred.await(launch))), - stop: Deferred.succeed(stopping, undefined).pipe(Effect.andThen(Deferred.await(stop))), - }); - yield* orchestrator.configure({ - members: [{ id: "api", activation: "lazy", idleMillis: 1000 }], - dependencies: [], - }); - yield* orchestrator.startComposition; - const first = yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.forkChild); - yield* Deferred.await(launching); - const second = yield* Effect.scoped(orchestrator.acquire("api")).pipe( - Effect.forkChild({ startImmediately: true }), - ); - yield* Deferred.succeed(launch, undefined); - yield* Fiber.join(first); - yield* Fiber.join(second); - expect(yield* Ref.get(instance.starts)).toHaveLength(1); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(stopping); - const third = yield* Effect.scoped(orchestrator.acquire("api")).pipe( - Effect.forkChild({ startImmediately: true }), - ); - yield* Deferred.succeed(stop, undefined); - yield* Fiber.join(third); - expect(yield* Ref.get(instance.starts)).toHaveLength(2); - yield* orchestrator.stopNamespace; - }), - ).pipe(Effect.provide(TestClock.layer())), -); - -it.live("allows later traffic to retry an armed service after a failed wake launch", () => - Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - const failing = yield* Ref.make(true); - const instance = yield* makeInstance(orchestrator, "api", { - launch: Ref.get(failing).pipe( - Effect.flatMap((value) => (value ? Effect.fail(failure("launch failed")) : Effect.void)), - ), - }); - yield* orchestrator.configure({ - members: [{ id: "api", activation: "lazy" }], - dependencies: [], - }); - yield* orchestrator.startComposition; - yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); - yield* Ref.set(failing, false); - yield* Effect.scoped(orchestrator.acquire("api")); - expect((yield* instance.core.get).health).toBe("healthy"); - yield* orchestrator.stopNamespace; - }), - ), -); - -it.live("logs a named failure for a failed wake and readiness for a successful one", () => { - const logs: Array = []; - return Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - const failing = yield* Ref.make(true); - yield* makeInstance(orchestrator, "api", { - launch: Ref.get(failing).pipe( - Effect.flatMap((value) => (value ? Effect.fail(failure("launch failed")) : Effect.void)), - ), - }); - yield* orchestrator.configure({ - members: [{ id: "api", activation: "lazy" }], - dependencies: [], - }); - yield* orchestrator.startComposition; - yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); - yield* Ref.set(failing, false); - yield* Effect.scoped(orchestrator.acquire("api")); - yield* orchestrator.stopNamespace; - expect(logs.some((line) => line.includes("api api failed to wake"))).toBe(true); - expect(logs.some((line) => line.includes("api api is ready"))).toBe(true); - }), - ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); -}); +describe("wake and idle sleep", () => { + it.live( + "does not let a prerequisite idle-sleep while its dependent is still preparing to wake", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const preparing = yield* Deferred.make(); + const continuePreparation = yield* Deferred.make(); + const database = yield* makeInstance(orchestrator, "database"); + const studio = yield* makeInstance(orchestrator, "studio", { + prepare: Deferred.succeed(preparing, undefined).pipe( + Effect.andThen(Deferred.await(continuePreparation)), + ), + }); + yield* orchestrator.configure({ + members: [ + { id: "database", activation: "lazy", idleMillis: 100 }, + { id: "studio", activation: "lazy" }, + ], + dependencies: [{ from: "database", to: "studio" }], + }); + yield* orchestrator.startComposition; + const wake = yield* Effect.scoped(orchestrator.acquire("studio")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.await(preparing); + expect((yield* database.status).lifecycle).toBe("running"); + yield* TestClock.adjust("1 second"); + expect((yield* database.status).lifecycle).toBe("running"); -it.live("logs exactly one wake for many concurrent acquires of a sleeping member", () => { - const logs: Array = []; - return Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - const launching = yield* Deferred.make(); - const proceed = yield* Deferred.make(); - yield* makeInstance(orchestrator, "api", { - launch: Deferred.succeed(launching, undefined).pipe( - Effect.andThen(Deferred.await(proceed)), - ), - }); - yield* orchestrator.configure({ - members: [{ id: "api", activation: "lazy" }], - dependencies: [], - }); - yield* orchestrator.startComposition; - const callers = yield* Effect.forEach(Array.from({ length: 20 }), () => - Effect.scoped(orchestrator.acquire("api")).pipe( - Effect.forkChild({ startImmediately: true }), - ), - ); - yield* Deferred.await(launching); - yield* Deferred.succeed(proceed, undefined); - yield* Effect.forEach(callers, Fiber.join); - yield* orchestrator.stopNamespace; - expect(logs.filter((line) => line.includes("Waking api api"))).toHaveLength(1); - expect(logs.filter((line) => line.includes("api api is ready"))).toHaveLength(1); - }), - ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); -}); + yield* Deferred.succeed(continuePreparation, undefined); + yield* Fiber.join(wake); + expect(yield* studio.status).toMatchObject({ lifecycle: "running", health: "healthy" }); + }), + ).pipe(Effect.provide(TestClock.layer())), + ); -it.live("logs a distinct failure when a launched member never becomes ready", () => { - const logs: Array = []; - return Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - yield* makeInstance(orchestrator, "api", { - health: Effect.fail(failure("still booting")), - }); - yield* orchestrator.configure({ - members: [{ id: "api", activation: "lazy" }], - dependencies: [], - }); - yield* orchestrator.startComposition; - yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); - yield* orchestrator.stopNamespace; - expect(logs.some((line) => line.includes("api api failed to become ready"))).toBe(true); - expect(logs.some((line) => line.includes("api api failed to wake"))).toBe(false); - }), - ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); -}); + it.live( + "does not idle-sleep a launch before its first healthy observation, then idles normally once healthy", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const health = yield* Deferred.make(); + const api = yield* makeInstance(orchestrator, "api", { + health: Deferred.await(health), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy", idleMillis: 100 }], + dependencies: [], + }); + yield* orchestrator.startComposition; + expect((yield* api.status).wakeEnabled).toBe(true); -it.live("logs no extra wake while the initiator is still waiting for readiness", () => { - const logs: Array = []; - return Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - const healthEntered = yield* Deferred.make(); - const readyGate = yield* Deferred.make(); - yield* makeInstance(orchestrator, "api", { - health: Deferred.succeed(healthEntered, undefined).pipe( - Effect.andThen(Deferred.await(readyGate)), - ), - }); - yield* orchestrator.configure({ - members: [{ id: "api", activation: "lazy" }], - dependencies: [], - }); - yield* orchestrator.startComposition; - const first = yield* Effect.scoped(orchestrator.acquire("api")).pipe( - Effect.forkChild({ startImmediately: true }), - ); - yield* Deferred.await(healthEntered); - expect(logs.filter((line) => line.includes("Waking api api"))).toHaveLength(1); - const second = yield* Effect.scoped(orchestrator.acquire("api")).pipe( - Effect.forkChild({ startImmediately: true }), - ); - yield* Deferred.succeed(readyGate, undefined); - yield* Fiber.join(first); - yield* Fiber.join(second); - yield* orchestrator.stopNamespace; - expect(logs.filter((line) => line.includes("Waking api api"))).toHaveLength(1); - expect(logs.filter((line) => line.includes("api api is ready"))).toHaveLength(1); - }), - ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); -}); + yield* orchestrator.start("api"); + expect(yield* api.status).toMatchObject({ lifecycle: "running", health: "starting" }); + yield* TestClock.adjust("1 second"); + expect((yield* api.status).lifecycle).toBe("running"); -it.live("does not log a spurious wake for a member already starting outside acquire", () => { - const logs: Array = []; - return Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - const launching = yield* Deferred.make(); - const proceed = yield* Deferred.make(); - yield* makeInstance(orchestrator, "api", { - launch: Deferred.succeed(launching, undefined).pipe( - Effect.andThen(Deferred.await(proceed)), - ), - }); - yield* orchestrator.configure({ - members: [{ id: "api", activation: "eager" }], - dependencies: [], - }); - const composition = yield* orchestrator.startComposition.pipe( - Effect.forkChild({ startImmediately: true }), - ); - yield* Deferred.await(launching); - const acquired = yield* Effect.scoped(orchestrator.acquire("api")).pipe( - Effect.forkChild({ startImmediately: true }), - ); - yield* Deferred.succeed(proceed, undefined); - yield* Fiber.join(composition); - yield* Fiber.join(acquired); - yield* orchestrator.stopNamespace; - expect(logs.filter((line) => line.includes("Waking api api"))).toHaveLength(0); - }), - ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); -}); + yield* Deferred.succeed(health, undefined); + yield* api.ready; + expect((yield* api.status).health).toBe("healthy"); + yield* TestClock.adjust("1 second"); + yield* stopped(api); + expect((yield* api.status).wakeEnabled).toBe(true); + }), + ).pipe(Effect.provide(TestClock.layer())), + ); -describe("readiness recovery", () => { - const recoverableHealth = (healthy: Ref.Ref) => - Ref.get(healthy).pipe( - Effect.flatMap((ok) => (ok ? Effect.void : Effect.fail(failure("still booting")))), - ); + it.live( + "holds an eager member's prerequisite active through a slow preparation instead of bypassing the reservation", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const preparing = yield* Deferred.make(); + const continuePreparation = yield* Deferred.make(); + yield* makeInstance(orchestrator, "database"); + const functions = yield* makeInstance(orchestrator, "functions"); + const studio = yield* makeInstance(orchestrator, "studio", { + prepare: Deferred.succeed(preparing, undefined).pipe( + Effect.andThen(Deferred.await(continuePreparation)), + ), + }); + yield* orchestrator.configure({ + members: [ + { id: "database", activation: "eager" }, + { id: "functions", activation: "lazy", idleMillis: 100 }, + { id: "studio", activation: "eager" }, + ], + dependencies: [ + { from: "database", to: "studio" }, + { from: "functions", to: "studio" }, + ], + }); + const composition = yield* orchestrator.startComposition.pipe(Effect.forkChild); + yield* Deferred.await(preparing); + + // Independent traffic wakes and releases `functions` while studio is still preparing. + yield* Effect.scoped(orchestrator.acquire("functions")); + expect((yield* functions.status).lifecycle).toBe("running"); + yield* TestClock.adjust("1 second"); + expect((yield* functions.status).lifecycle).toBe("running"); + + yield* Deferred.succeed(continuePreparation, undefined); + yield* Fiber.join(composition); + expect(yield* studio.status).toMatchObject({ lifecycle: "running", health: "healthy" }); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live("holds a restarted member's prerequisite active through its own slow preparation", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const preparing = yield* Deferred.make(); + const continuePreparation = yield* Deferred.make(); + yield* makeInstance(orchestrator, "database"); + const functions = yield* makeInstance(orchestrator, "functions"); + const studio = yield* makeInstance(orchestrator, "studio", { + prepare: Deferred.succeed(preparing, undefined).pipe( + Effect.andThen(Deferred.await(continuePreparation)), + ), + }); + yield* orchestrator.configure({ + members: [ + { id: "database", activation: "lazy" }, + { id: "functions", activation: "lazy", idleMillis: 100 }, + { id: "studio", activation: "lazy" }, + ], + dependencies: [ + { from: "database", to: "studio" }, + { from: "functions", to: "studio" }, + ], + }); + yield* orchestrator.startComposition; + // Functions is independently already running; studio itself is still asleep, so nothing + // but studio's own restart keeps functions awake while studio prepares. + yield* Effect.scoped(orchestrator.acquire("functions")); + expect((yield* functions.status).lifecycle).toBe("running"); + expect((yield* studio.status).lifecycle).toBe("stopped"); + + const restart = yield* orchestrator + .restart("studio") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(preparing); + yield* TestClock.adjust("1 second"); + expect((yield* functions.status).lifecycle).toBe("running"); + + yield* Deferred.succeed(continuePreparation, undefined); + yield* Fiber.join(restart); + expect(yield* studio.status).toMatchObject({ lifecycle: "running", health: "healthy" }); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live("keeps every prerequisite of a running dependent awake past its own idle timeout", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const pgmeta = yield* makeInstance(orchestrator, "pgmeta"); + const functions = yield* makeInstance(orchestrator, "functions"); + const studio = yield* makeInstance(orchestrator, "studio"); + yield* orchestrator.configure({ + members: [ + { id: "pgmeta", activation: "lazy", idleMillis: 60_000 }, + { id: "functions", activation: "lazy", idleMillis: 60_000 }, + { id: "studio", activation: "lazy", idleMillis: 300_000 }, + ], + dependencies: [ + { from: "pgmeta", to: "studio" }, + { from: "functions", to: "studio" }, + ], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("studio")); + + yield* TestClock.adjust("299 seconds"); + expect((yield* studio.status).lifecycle).toBe("running"); + expect((yield* pgmeta.status).lifecycle).toBe("running"); + expect((yield* functions.status).lifecycle).toBe("running"); + + const studioStopped = yield* stopped(studio).pipe(Effect.forkChild); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(studioStopped); + expect((yield* pgmeta.status).lifecycle).toBe("running"); + expect((yield* functions.status).lifecycle).toBe("running"); + + const prerequisitesStopped = yield* Effect.all([stopped(pgmeta), stopped(functions)], { + concurrency: "unbounded", + }).pipe(Effect.forkChild); + yield* TestClock.adjust("60 seconds"); + yield* Fiber.join(prerequisitesStopped); + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live("keeps serving traffic admitted before an idle timer fires, without a restart", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const api = yield* makeInstance(orchestrator, "api"); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy", idleMillis: 1000 }], + dependencies: [], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("api")); + yield* TestClock.adjust("999 millis"); + + const request = yield* Scope.make(); + yield* orchestrator.acquire("api").pipe(Scope.provide(request)); + yield* TestClock.adjust("10 seconds"); + expect((yield* api.status).lifecycle).toBe("running"); + expect(yield* Ref.get(api.starts)).toHaveLength(1); + + const sleeping = yield* stopped(api).pipe(Effect.forkChild); + yield* Scope.close(request, Exit.void); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(sleeping); + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live( + "holds traffic that arrives after a committed sleep until the stop is confirmed, then serves it from the next launch", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const stopping = yield* Deferred.make(); + const finishStop = yield* Deferred.make(); + const api = yield* makeInstance(orchestrator, "api", { + stop: Deferred.succeed(stopping, undefined).pipe( + Effect.andThen(Deferred.await(finishStop)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy", idleMillis: 1000 }], + dependencies: [], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("api")); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(stopping); + + const late = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + expect((yield* api.status).lifecycle).toBe("stopping"); + expect(yield* Ref.get(api.starts)).toHaveLength(1); + + yield* Deferred.succeed(finishStop, undefined); + yield* Fiber.join(late); + expect(yield* Ref.get(api.starts)).toHaveLength(2); + expect(yield* api.status).toMatchObject({ + lifecycle: "running", + health: "healthy", + launchId: 2, + }); + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live( + "fails only a waiter whose wake budget expires, naming the stage, while the shared launch continues", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const preparing = yield* Deferred.make(); + const continuePreparation = yield* Deferred.make(); + const api = yield* makeInstance(orchestrator, "api", { + prepare: Deferred.succeed(preparing, undefined).pipe( + Effect.andThen(Deferred.await(continuePreparation)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + + const early = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.flip, + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.await(preparing); + yield* TestClock.adjust("60 seconds"); + const late = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* TestClock.adjust("60 seconds"); + + const expired = yield* Fiber.join(early); + expect(expired.message).toBe( + "api wake budget exceeded while waiting for api to finish preparing", + ); + expect((yield* api.status).lifecycle).toBe("starting"); + + yield* Deferred.succeed(continuePreparation, undefined); + yield* Fiber.join(late); + expect(yield* Ref.get(api.starts)).toHaveLength(1); + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live("releases only a cancelled waiter and keeps the shared launch for the others", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const launching = yield* Deferred.make(); + const proceed = yield* Deferred.make(); + const api = yield* makeInstance(orchestrator, "api", { + launch: Deferred.succeed(launching, undefined).pipe( + Effect.andThen(Deferred.await(proceed)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const cancelled = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + const kept = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.await(launching); + yield* Fiber.interrupt(cancelled); + + yield* Deferred.succeed(proceed, undefined); + yield* Fiber.join(kept); + expect(yield* api.status).toMatchObject({ lifecycle: "running", health: "healthy" }); + expect(yield* Ref.get(api.starts)).toHaveLength(1); + yield* orchestrator.stopNamespace; + }), + ), + ); + + it.live("joins an in-progress wake and wakes again after an admitted idle stop", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const launching = yield* Deferred.make(); + const launch = yield* Deferred.make(); + const stopping = yield* Deferred.make(); + const stop = yield* Deferred.make(); + const instance = yield* makeInstance(orchestrator, "api", { + launch: Deferred.succeed(launching, undefined).pipe( + Effect.andThen(Deferred.await(launch)), + ), + stop: Deferred.succeed(stopping, undefined).pipe(Effect.andThen(Deferred.await(stop))), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy", idleMillis: 1000 }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const first = yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.forkChild); + yield* Deferred.await(launching); + const second = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(launch, undefined); + yield* Fiber.join(first); + yield* Fiber.join(second); + expect(yield* Ref.get(instance.starts)).toHaveLength(1); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(stopping); + const third = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(stop, undefined); + yield* Fiber.join(third); + expect(yield* Ref.get(instance.starts)).toHaveLength(2); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); +}); + +describe("idle races through a real listener", () => { + interface HeldResponse { + readonly release: () => void; + } + + /** + * A lazy service whose every generation is a real HTTP server answering `generation:N`, reached + * only through its namespace listener. `/hold` keeps the response open until released, and a + * stop can be held open the same way. + */ + const makeListenerStack = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "orchestrator-listener-" }); + const stackId = `listener-${root.split("/").at(-1) ?? "stack"}`; + const state = yield* Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), + ); + yield* state.save({ + id: stackId, + identity: { projectRoot: root, branchContext: "test", stackName: stackId }, + runtime: "native", + lifetime: "detached", + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }); + const network = yield* Layer.build( + Network.layer({ stackId, runtime: "native" }).pipe( + Layer.provide(Layer.succeed(StackNamespace.Service, state)), + ), + ).pipe(Effect.map((context) => Context.get(context, Network.Service))); + const orchestrator = yield* makeTestOrchestrator(); + const address = yield* Ref.make<{ readonly host: string; readonly port: number } | undefined>( + undefined, + ); + const generations = yield* Ref.make(0); + const held: Array = []; + const holdEntered = yield* Deferred.make(); + const holdStops = yield* Ref.make< + | { readonly entered: Deferred.Deferred; readonly release: Deferred.Deferred } + | undefined + >(undefined); + + const core = yield* makeService>( + { + launch: () => + Effect.gen(function* () { + const generation = yield* Ref.updateAndGet(generations, (value) => value + 1); + const server = createServer((request, response) => { + const answer = () => response.end(`generation:${generation}`); + if (request.url !== "/hold") return answer(); + held.push({ release: answer }); + Deferred.doneUnsafe(holdEntered, Effect.void); + }); + const port = yield* Effect.callback((resume) => { + server.listen(0, "127.0.0.1", () => { + const bound = server.address(); + resume( + bound === null || typeof bound === "string" + ? Effect.fail(failure("no address")) + : Effect.succeed(bound.port), + ); + }); + }); + yield* Ref.set(address, { host: "127.0.0.1", port }); + const exited = yield* Deferred.make>(); + const close = Effect.callback((resume) => { + server.close(() => resume(Effect.void)); + server.closeAllConnections(); + }); + return { + health: Effect.void, + exit: Deferred.await(exited), + stop: Ref.get(holdStops).pipe( + Effect.flatMap((gate) => + gate === undefined + ? Effect.void + : Deferred.succeed(gate.entered, undefined).pipe( + Effect.andThen(Deferred.await(gate.release)), + ), + ), + Effect.andThen(close), + Effect.andThen(Deferred.succeed(exited, Exit.void)), + Effect.asVoid, + ), + remove: Effect.void, + }; + }), + removeData: () => Effect.void, + }, + { id: "api", config: {}, report: orchestrator.report }, + ); + const namespace = yield* network.register({ + id: "api", + endpoints: { + http: { + protocol: "http", + port: "auto", + backend: orchestrator.acquire("api").pipe( + Effect.andThen(Ref.get(address)), + Effect.flatMap((current) => + current === undefined ? Effect.fail(failure("no session")) : Effect.succeed(current), + ), + Effect.mapError((cause) => new ProxyError({ message: cause.message, cause })), + ), + enabled: Effect.succeed(true), + }, + }, + }); + yield* orchestrator.register({ + id: "api", + service: "api", + core, + launch: (generation, inputs) => core.launch(generation, inputs), + prepare: () => Effect.void, + bind: namespace.bind.pipe( + Effect.asVoid, + Effect.mapError((cause) => failure(cause.message)), + ), + close: Effect.void, + hasEndpoint: true, + inputs: [], + outputs: {}, + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy", idleMillis: 1000 }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const [binding] = yield* namespace.bindings; + if (binding === undefined) return yield* Effect.die("listener was not bound"); + const get = (path: string) => + Effect.gen(function* () { + const client = yield* HttpClient.HttpClient; + const response = yield* client.get(`http://127.0.0.1:${binding.port}${path}`); + return { status: response.status, body: yield* response.text }; + }).pipe(Effect.provide(NodeHttpClient.layerNodeHttp)); + const releaseHeld = Effect.sync(() => { + for (const response of held.splice(0)) response.release(); + }); + return { + orchestrator, + get, + holdEntered, + releaseHeld, + holdStops, + status: orchestrator.status("api"), + observation: orchestrator.changes("api"), + }; + }); + + it.live( + "keeps serving a request admitted before the idle timer fires, from the same generation", + () => + Effect.scoped( + Effect.gen(function* () { + const stack = yield* makeListenerStack; + // A scoped acquisition releases its lease before returning, so the idle timer is armed. + yield* Effect.scoped(stack.orchestrator.acquire("api")); + + const request = yield* stack + .get("/hold") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(stack.holdEntered); + yield* TestClock.adjust("10 seconds"); + expect(yield* stack.status).toMatchObject({ lifecycle: "running", launchId: 1 }); + + yield* stack.releaseHeld; + expect(yield* Fiber.join(request)).toEqual({ status: 200, body: "generation:1" }); + yield* stack.orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, TestClock.layer()))), + ); + + it.live( + "holds a request arriving after a committed sleep until the stop is confirmed, then serves it from the next generation", + () => + Effect.scoped( + Effect.gen(function* () { + const stack = yield* makeListenerStack; + // A scoped acquisition releases its lease before returning, so the idle timer is armed. + yield* Effect.scoped(stack.orchestrator.acquire("api")); + const gate = { + entered: yield* Deferred.make(), + release: yield* Deferred.make(), + }; + yield* Ref.set(stack.holdStops, gate); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(gate.entered); + expect((yield* stack.status).lifecycle).toBe("stopping"); + + // The request's own admission is the next lifecycle transition while the stop is held. + const queued = yield* nextTransition(stack.observation); + const request = yield* stack.get("/").pipe(Effect.forkChild({ startImmediately: true })); + yield* Fiber.join(queued); + yield* Ref.set(stack.holdStops, undefined); + yield* Deferred.succeed(gate.release, undefined); + + expect(yield* Fiber.join(request)).toEqual({ status: 200, body: "generation:2" }); + expect(yield* stack.status).toMatchObject({ lifecycle: "running", launchId: 2 }); + yield* stack.orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, TestClock.layer()))), + ); +}); + +describe("admission while the lifecycle gate is held", () => { + // A configure whose persistence never finishes holds the gate the way a stalled state write does. + const holdGate = (orchestrator: Orchestrator.Interface) => + Effect.gen(function* () { + const persisting = yield* Deferred.make(); + const release = yield* Deferred.make(); + const configuring = yield* orchestrator + .configure( + { members: [{ id: "api", activation: "lazy" }], dependencies: [] }, + Deferred.succeed(persisting, undefined).pipe(Effect.andThen(Deferred.await(release))), + ) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(persisting); + return { release: Deferred.succeed(release, undefined), configuring }; + }); + + it.live("lets a client cancel an acquisition still waiting for the gate", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + yield* makeInstance(orchestrator, "api"); + const held = yield* holdGate(orchestrator); + const waiting = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + const cancelled = yield* Fiber.interrupt(waiting).pipe(Effect.timeoutOption("2 seconds")); + expect(Option.isSome(cancelled)).toBe(true); + yield* held.release; + yield* Fiber.join(held.configuring); + }), + ), + ); + + it.live("fails an acquisition whose wake budget runs out while it waits for the gate", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + yield* makeInstance(orchestrator, "api"); + const held = yield* holdGate(orchestrator); + const waiting = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.flip, + Effect.forkChild({ startImmediately: true }), + ); + yield* TestClock.adjust("120 seconds"); + yield* held.release; + yield* Fiber.join(held.configuring); + expect((yield* Fiber.join(waiting)).message).toBe( + "api wake budget exceeded while waiting for admission", + ); + }), + ).pipe(Effect.provide(TestClock.layer())), + ); +}); + +describe("failed and crashed services", () => { + it.live("allows later traffic to retry an armed service after a failed wake launch", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const failing = yield* Ref.make(true); + const instance = yield* makeInstance(orchestrator, "api", { + launch: Ref.get(failing).pipe( + Effect.flatMap((value) => + value ? Effect.fail(failure("launch failed")) : Effect.void, + ), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); + yield* Ref.set(failing, false); + yield* Effect.scoped(orchestrator.acquire("api")); + expect((yield* instance.status).health).toBe("healthy"); + yield* orchestrator.stopNamespace; + }), + ), + ); + + it.live( + "keeps a crashed armed service wakeable and wakes it exactly once on the next request", + () => { + const logs: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const crash = yield* Deferred.make>(); + const api = yield* makeInstance(orchestrator, "api", { exit: crash }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("api")); + expect(yield* api.status).toMatchObject({ lifecycle: "running", health: "healthy" }); + + // The runtime exits on its own, with no stop/restart/destroy ever invalidating the intent. + yield* Deferred.succeed( + crash, + Exit.fail(new ServiceError({ operation: "process", message: "crashed" })), + ); + yield* stopped(api); + expect((yield* api.status).wakeEnabled).toBe(true); + + const recoveryLogStart = logs.length; + const callers = yield* Effect.forEach(Array.from({ length: 5 }), () => + Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ), + ); + yield* Effect.forEach(callers, Fiber.join); + expect(yield* api.status).toMatchObject({ lifecycle: "running", health: "healthy" }); + yield* orchestrator.stopNamespace; + expect( + logs.slice(recoveryLogStart).filter((line) => line.includes("Waking api api")), + ).toHaveLength(1); + }), + ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); + }, + ); + + it.live( + "opens the breaker after three crashes, fails fast naming the cause, doubles the cooldown on reopening, and resets on an explicit start", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const crashes = yield* Queue.unbounded>(); + const api = yield* makeInstance(orchestrator, "api", { crash: Queue.take(crashes) }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const segfault = Exit.fail( + new ServiceError({ operation: "process", message: "segfault" }), + ); + const serveThenCrash = Effect.gen(function* () { + yield* Effect.scoped(orchestrator.acquire("api")); + const down = yield* stopped(api).pipe(Effect.forkChild({ startImmediately: true })); + yield* Queue.offer(crashes, segfault); + yield* Fiber.join(down); + }); + + yield* serveThenCrash; + yield* serveThenCrash; + yield* serveThenCrash; + const fastFail = yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); + expect(fastFail.message).toBe("api circuit breaker is open: segfault"); + expect(yield* Ref.get(api.starts)).toHaveLength(3); + + // The cooldown's own state change is the only signal that it elapsed. + const coolDown = (duration: Duration.Input) => + Effect.gen(function* () { + const cooled = yield* nextTransition(api.observation); + yield* TestClock.adjust(duration); + yield* Fiber.join(cooled); + }); + yield* coolDown("30 seconds"); + yield* serveThenCrash; + expect(yield* Ref.get(api.starts)).toHaveLength(4); + + // Reopening doubles the cooldown: 30 seconds no longer suffice. + yield* TestClock.adjust("30 seconds"); + expect( + (yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip)).message, + ).toBe("api circuit breaker is open: segfault"); + yield* coolDown("30 seconds"); + yield* serveThenCrash; + expect(yield* Ref.get(api.starts)).toHaveLength(5); + yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); + + yield* orchestrator.start("api"); + expect((yield* api.status).lifecycle).toBe("running"); + expect(yield* Ref.get(api.starts)).toHaveLength(6); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live("counts a crash toward the breaker even when its cleanup has to be retried", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const crashes = yield* Queue.unbounded>(); + const stops = yield* Ref.make(0); + const api = yield* makeInstance(orchestrator, "api", { + crash: Queue.take(crashes), + // Every generation's first cleanup attempt fails; the retry succeeds. + stop: Ref.updateAndGet(stops, (count) => count + 1).pipe( + Effect.flatMap((count) => + count % 2 === 1 ? Effect.fail(failure("container busy")) : Effect.void, + ), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const serveThenCrash = Effect.gen(function* () { + yield* Effect.scoped(orchestrator.acquire("api")); + const cleanupPending = yield* api.observation.pipe( + Stream.filter( + (state) => + state.lifecycle === "stopping" && + state.currentOperation === undefined && + state.cleanupError !== undefined, + ), + Stream.take(1), + Stream.runDrain, + Effect.forkChild({ startImmediately: true }), + ); + yield* Queue.offer(crashes, Exit.fail(failure("segfault"))); + yield* Fiber.join(cleanupPending); + }); + + yield* serveThenCrash; + yield* serveThenCrash; + yield* serveThenCrash; + const fourth = yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.exit); + expect(errorOf(fourth)?.message).toBe("api circuit breaker is open: segfault"); + expect(yield* Ref.get(api.starts)).toHaveLength(3); + }), + ), + ); + + it.live("forgets earlier crashes once a generation stays healthy for 30 seconds", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const crashes = yield* Queue.unbounded>(); + const api = yield* makeInstance(orchestrator, "api", { crash: Queue.take(crashes) }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const crash = Effect.gen(function* () { + const down = yield* stopped(api).pipe(Effect.forkChild({ startImmediately: true })); + yield* Queue.offer(crashes, Exit.fail(failure("segfault"))); + yield* Fiber.join(down); + }); + + yield* Effect.scoped(orchestrator.acquire("api")); + yield* crash; + yield* Effect.scoped(orchestrator.acquire("api")); + yield* crash; + yield* Effect.scoped(orchestrator.acquire("api")); + yield* TestClock.adjust("30 seconds"); + yield* crash; + + // Without the reset this would be the third consecutive failure, opening the breaker. + yield* Effect.scoped(orchestrator.acquire("api")); + expect(yield* Ref.get(api.starts)).toHaveLength(4); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live("relaunches a crashed prerequisite without replacing its healthy dependent", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const crash = yield* Deferred.make>(); + const database = yield* makeInstance(orchestrator, "database", { exit: crash }); + const rest = yield* makeInstance(orchestrator, "rest"); + yield* orchestrator.configure({ + members: [ + { id: "database", activation: "lazy" }, + { id: "rest", activation: "lazy" }, + ], + dependencies: [{ from: "database", to: "rest" }], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("rest")); + + const relaunched = yield* database.observation.pipe( + Stream.filter((state) => state.launchId === 2 && state.health === "healthy"), + Stream.take(1), + Stream.runDrain, + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(crash, Exit.fail(failure("segfault"))); + yield* Fiber.join(relaunched); + + expect(yield* rest.status).toMatchObject({ lifecycle: "running", launchId: 1 }); + expect(yield* Ref.get(rest.starts)).toHaveLength(1); + yield* Effect.scoped(orchestrator.acquire("rest")); + yield* orchestrator.stopNamespace; + }), + ), + ); + + it.live( + "rejects traffic beyond 256 waiters and frees a slot when a waiter is cancelled or expires", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const preparing = yield* Deferred.make(); + const continuePreparation = yield* Deferred.make(); + yield* makeInstance(orchestrator, "api", { + prepare: Deferred.succeed(preparing, undefined).pipe( + Effect.andThen(Deferred.await(continuePreparation)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const waitFor = Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.flip, + Effect.forkChild({ startImmediately: true }), + ); + const queued = yield* Effect.forEach(Array.from({ length: 256 }), () => waitFor); + yield* Deferred.await(preparing); + expect((yield* Fiber.join(yield* waitFor)).message).toBe("api has too many waiters"); + + const [cancelled] = queued; + if (cancelled !== undefined) yield* Fiber.interrupt(cancelled); + const replacement = yield* waitFor; + yield* TestClock.adjust("120 seconds"); + expect((yield* Fiber.join(replacement)).message).toContain("wake budget exceeded"); + + const afterExpiry = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(continuePreparation, undefined); + yield* Fiber.join(afterExpiry); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); +}); + +describe("wake logging", () => { + it.live("logs a named failure for a failed wake and readiness for a successful one", () => { + const logs: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const failing = yield* Ref.make(true); + yield* makeInstance(orchestrator, "api", { + launch: Ref.get(failing).pipe( + Effect.flatMap((value) => + value ? Effect.fail(failure("launch failed")) : Effect.void, + ), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); + yield* Ref.set(failing, false); + yield* Effect.scoped(orchestrator.acquire("api")); + yield* orchestrator.stopNamespace; + expect(logs.some((line) => line.includes("api api failed to wake"))).toBe(true); + expect(logs.some((line) => line.includes("api api is ready"))).toBe(true); + }), + ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); + }); + + it.live("logs exactly one wake for many concurrent acquires of a sleeping member", () => { + const logs: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const launching = yield* Deferred.make(); + const proceed = yield* Deferred.make(); + yield* makeInstance(orchestrator, "api", { + launch: Deferred.succeed(launching, undefined).pipe( + Effect.andThen(Deferred.await(proceed)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const callers = yield* Effect.forEach(Array.from({ length: 20 }), () => + Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ), + ); + yield* Deferred.await(launching); + yield* Deferred.succeed(proceed, undefined); + yield* Effect.forEach(callers, Fiber.join); + yield* orchestrator.stopNamespace; + expect(logs.filter((line) => line.includes("Waking api api"))).toHaveLength(1); + expect(logs.filter((line) => line.includes("api api is ready"))).toHaveLength(1); + }), + ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); + }); + + it.live("logs a distinct failure when a launched member never becomes ready", () => { + const logs: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + yield* makeInstance(orchestrator, "api", { + health: Effect.fail(failure("still booting")), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); + yield* orchestrator.stopNamespace; + expect(logs.some((line) => line.includes("api api failed to become ready"))).toBe(true); + expect(logs.some((line) => line.includes("api api failed to wake"))).toBe(false); + }), + ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); + }); + + it.live("logs no extra wake while the initiator is still waiting for readiness", () => { + const logs: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const healthEntered = yield* Deferred.make(); + const readyGate = yield* Deferred.make(); + yield* makeInstance(orchestrator, "api", { + health: Deferred.succeed(healthEntered, undefined).pipe( + Effect.andThen(Deferred.await(readyGate)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const first = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.await(healthEntered); + expect(logs.filter((line) => line.includes("Waking api api"))).toHaveLength(1); + const second = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(readyGate, undefined); + yield* Fiber.join(first); + yield* Fiber.join(second); + yield* orchestrator.stopNamespace; + expect(logs.filter((line) => line.includes("Waking api api"))).toHaveLength(1); + expect(logs.filter((line) => line.includes("api api is ready"))).toHaveLength(1); + }), + ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); + }); + + it.live("does not log a spurious wake for a member already starting outside acquire", () => { + const logs: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const launching = yield* Deferred.make(); + const proceed = yield* Deferred.make(); + yield* makeInstance(orchestrator, "api", { + launch: Deferred.succeed(launching, undefined).pipe( + Effect.andThen(Deferred.await(proceed)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "eager" }], + dependencies: [], + }); + const composition = yield* orchestrator.startComposition.pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.await(launching); + const acquired = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(proceed, undefined); + yield* Fiber.join(composition); + yield* Fiber.join(acquired); + yield* orchestrator.stopNamespace; + expect(logs.filter((line) => line.includes("Waking api api"))).toHaveLength(0); + }), + ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); + }); +}); + +describe("readiness recovery", () => { + const recoverableHealth = (healthy: Ref.Ref) => + Ref.get(healthy).pipe( + Effect.flatMap((ok) => (ok ? Effect.void : Effect.fail(failure("still booting")))), + ); + + it.live( + "keeps re-checking a slow prerequisite after a failed reprobe until a waiting dependent is served", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const probes = yield* Ref.make(0); + const firstProbe = yield* Deferred.make(); + const failFirstProbe = yield* Deferred.make(); + const analytics = yield* makeInstance(orchestrator, "analytics", { + health: Effect.fail(failure("still booting")), + probe: Ref.updateAndGet(probes, (count) => count + 1).pipe( + Effect.flatMap((count) => + count === 1 + ? Deferred.succeed(firstProbe, undefined).pipe( + Effect.andThen(Deferred.await(failFirstProbe)), + Effect.andThen(Effect.fail(failure("still booting"))), + ) + : Effect.void, + ), + ), + }); + const studio = yield* makeInstance(orchestrator, "studio"); + yield* orchestrator.configure({ + members: [ + { id: "analytics", activation: "lazy" }, + { id: "studio", activation: "lazy" }, + ], + dependencies: [{ from: "analytics", to: "studio" }], + }); + yield* orchestrator.startComposition; + + const request = yield* Effect.scoped(orchestrator.acquire("studio")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.await(firstProbe); + const failed = yield* nextTransition(analytics.observation); + yield* Deferred.succeed(failFirstProbe, undefined); + yield* Fiber.join(failed); + yield* TestClock.adjust("1 second"); + + yield* Fiber.join(request); + expect(yield* Ref.get(probes)).toBe(2); + expect(yield* Ref.get(analytics.starts)).toHaveLength(1); + expect(yield* studio.status).toMatchObject({ lifecycle: "running", health: "healthy" }); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); it.live("serves traffic once a running instance recovers without restarting it", () => Effect.scoped( @@ -957,7 +1901,7 @@ describe("readiness recovery", () => { yield* Effect.scoped(orchestrator.acquire("api")); expect(yield* Ref.get(api.starts)).toHaveLength(1); - expect(yield* api.core.get).toMatchObject({ lifecycle: "running", health: "healthy" }); + expect(yield* api.status).toMatchObject({ lifecycle: "running", health: "healthy" }); yield* orchestrator.stopNamespace; }), ), @@ -988,7 +1932,7 @@ describe("readiness recovery", () => { expect(yield* Ref.get(database.starts)).toHaveLength(1); expect(yield* Ref.get(rest.starts)).toHaveLength(1); - expect((yield* rest.core.get).health).toBe("healthy"); + expect((yield* rest.status).health).toBe("healthy"); yield* orchestrator.stopNamespace; }), ), diff --git a/packages/stack/src/Orchestrator.ts b/packages/stack/src/Orchestrator.ts index 2ae7d33c58..93265bf985 100644 --- a/packages/stack/src/Orchestrator.ts +++ b/packages/stack/src/Orchestrator.ts @@ -6,35 +6,36 @@ import { Effect, Exit, FiberMap, - Match, + Option, Ref, Schema, Scope, Semaphore, Stream, + SubscriptionRef, } from "effect"; import { - ServiceError, - ServiceDestroyed, - ServiceNotRunning, - ServiceStaleLaunch, - type ServiceObservation, -} from "./Service.ts"; -import type { ServiceAdmission } from "./Service.ts"; + initialState, + LifecycleEvent, + makeGraph, + reduce, + waiterBudgetMillis, + type LifecycleCommand, + type LifecycleGraph, + type LifecycleState, + type ServiceSpec, +} from "./Lifecycle.ts"; +import { ServiceError, type ServiceInstance, type ServiceObservation } from "./Service.ts"; import { errorChainMessage } from "./internal/error-message.ts"; export type OrchestratorOperation = | "start" | "stop" | "restart" - | "sleep" | "destroy" - | "arm" - | "bind" - | "close" | "configure" | "register" - | "proxy"; + | "storage"; export class OrchestratorError extends Data.TaggedError("OrchestratorError")<{ readonly operation: OrchestratorOperation; @@ -42,7 +43,7 @@ export class OrchestratorError extends Data.TaggedError("OrchestratorError")<{ readonly cause?: unknown; readonly outcomes?: ReadonlyArray<{ readonly id: string; - readonly result: Exit.Exit; + readonly result: Exit.Exit; }>; }> {} @@ -50,38 +51,27 @@ export class OrchestratorError extends Data.TaggedError("OrchestratorError")<{ export const causeMessage = (cause: Cause.Cause): string => Cause.prettyErrors(cause).map(errorChainMessage).join("; ") || "interrupted"; -type CoreObservation = ServiceObservation; - -interface RegisteredCore { - readonly get: Effect.Effect; - readonly ready: Effect.Effect; - readonly stop: Effect.Effect; - readonly destroy: Effect.Effect; - readonly armAt: ( - revision: number, - guard?: Effect.Effect, - ) => Effect.Effect; - readonly sleep: Effect.Effect; - readonly observation: Stream.Stream; -} +type ExecutionCore = Pick< + ServiceInstance, + "get" | "observation" | "stop" | "reprobe" | "storage" | "removeData" +>; export interface RegisteredInstance { readonly id: string; /** The service kind this instance runs, for wake observability; the id is the graph identity. */ readonly service: string; - readonly core: RegisteredCore; - readonly startAt: ( - revision: number, + readonly core: ExecutionCore; + /** Resolves the launch configuration from bound inputs and an optional restart candidate, then launches. */ + readonly launch: ( + generation: number, inputs: Record, - wake: boolean, - guard?: Effect.Effect, - ) => Effect.Effect; - readonly restart: ( - revision: number, + candidate: unknown, + ) => Effect.Effect; + /** Resolves the same configuration as `launch` and only prepares it. */ + readonly prepare: ( inputs: Record, - config?: unknown, - guard?: Effect.Effect, - ) => Effect.Effect; + candidate: unknown, + ) => Effect.Effect; readonly bind: Effect.Effect; readonly close: Effect.Effect; readonly hasEndpoint: boolean; @@ -89,12 +79,6 @@ export interface RegisteredInstance { readonly outputs: Readonly>>; } -export type LifecycleError = - | ServiceError - | ServiceDestroyed - | ServiceNotRunning - | ServiceStaleLaunch; - interface CompositionMember { readonly id: string; readonly activation: "eager" | "lazy"; @@ -136,80 +120,85 @@ export const CompositionConfig = Schema.Struct({ ), }); +/** One service's lifecycle phase and health from the reducer, joined with its execution facts. */ +export interface Status extends ServiceObservation { + readonly lifecycle: "stopped" | "starting" | "running" | "stopping"; + readonly health: "starting" | "healthy" | "unhealthy" | undefined; + /** The live generation, if any. */ + readonly launchId: number | undefined; + /** Whether demand (traffic, a dependent or eager intent) relaunches the service. */ + readonly wakeEnabled: boolean; + /** Whether a destroy is waiting for the service to stop and its storage to be free, or removing it. */ + readonly destroyPending: boolean; +} + +/** An explicit operation the owner may refuse, for example while it drains. */ +export type AdmittedOperation = "start" | "restart" | "storage"; + /** The lifecycle authority for one stack's registered instances and their composition. */ export interface Interface { - readonly admissionFor: ( - id: string, - ) => ( - operation: ServiceAdmission, - transition: Effect.Effect, - ) => Effect.Effect; + /** Delivers a service's execution outcome to the lifecycle reducer. */ + readonly report: (event: LifecycleEvent) => Effect.Effect; readonly register: (entry: Entry) => Effect.Effect; readonly get: (id: string) => Effect.Effect; + readonly status: (id: string) => Effect.Effect; + readonly changes: (id: string) => Stream.Stream; readonly composition: Effect.Effect; /** * Validates and installs a composition; a failed `persist` keeps the previous one. - * `persist` runs under the graph gate, so callers take the cross-process state lock first. + * `persist` runs under the lifecycle gate, so callers take the cross-process state lock first. */ readonly configure: ( configuration: CompositionConfig, persist?: Effect.Effect, ) => Effect.Effect; - readonly start: (id: string) => Effect.Effect; - readonly stop: (id: string) => Effect.Effect; + readonly start: (id: string) => Effect.Effect; + readonly ready: (id: string) => Effect.Effect; + readonly stop: (id: string) => Effect.Effect; readonly restart: ( id: string, config?: unknown, - ) => Effect.Effect; - readonly destroy: (id: string) => Effect.Effect; - readonly startComposition: Effect.Effect< - ReadonlyArray, - OrchestratorError | LifecycleError - >; - readonly stopComposition: Effect.Effect< - ReadonlyArray, - OrchestratorError | LifecycleError - >; + ) => Effect.Effect; + readonly destroy: (id: string) => Effect.Effect; + /** Runs storage work while the lifecycle keeps the stopped service from launching. */ + readonly storage: ( + id: string, + operation: Effect.Effect, + ) => Effect.Effect; + readonly startComposition: Effect.Effect, OrchestratorError | ServiceError>; + readonly stopComposition: Effect.Effect, OrchestratorError | ServiceError>; readonly restartComposition: Effect.Effect< - ReadonlyArray, - OrchestratorError | LifecycleError + ReadonlyArray, + OrchestratorError | ServiceError >; - readonly stopNamespace: Effect.Effect; - readonly destroyNamespace: Effect.Effect; + readonly stopNamespace: Effect.Effect; + readonly destroyNamespace: Effect.Effect; + /** Admits one unit of traffic, waking the service if needed; the lease lasts for the scope. */ readonly acquire: ( id: string, awaitReady?: boolean, trigger?: string, - ) => Effect.Effect; -} - -interface ActivityState { - readonly active: number; - readonly lastActivity: number; + ) => Effect.Effect; } interface Graph { readonly members: ReadonlyMap; readonly prerequisites: ReadonlyMap>; - readonly dependents: ReadonlyMap>; } -interface StartPlan { - readonly order: ReadonlyArray; - readonly revisions: ReadonlyMap; - readonly structure: Graph; - readonly configuration: CompositionConfig; +interface Waiter { + readonly deferred: Deferred.Deferred; } -const serviceFailure = (operation: string, message: string): ServiceError => - new ServiceError({ operation, message }); +interface Applied { + readonly before: LifecycleState; + readonly after: LifecycleState; + readonly commands: ReadonlyArray; +} const graphError = (operation: OrchestratorOperation, message: string, cause?: unknown) => new OrchestratorError({ operation, message, ...(cause === undefined ? {} : { cause }) }); -const isActive = (observation: CoreObservation): boolean => - observation.lifecycle !== "stopped" || observation.wakeEnabled; - const topo = ( ids: ReadonlyArray, prerequisites: ReadonlyMap>, @@ -226,23 +215,186 @@ const topo = ( return result; }; -/** Builds an orchestrator whose watchers and idle timers live in the current scope. */ -export const make = Effect.fn("Orchestrator.make")(function* < - Entry extends RegisteredInstance, ->(): Effect.fn.Return, never, Scope.Scope> { +const validateGraph = ( + configuration: CompositionConfig, + values: ReadonlyMap, +): Graph | OrchestratorError => { + const members = new Map(); + for (const member of configuration.members) { + if (members.has(member.id)) return graphError("configure", `Duplicate member ${member.id}`); + members.set(member.id, member); + } + const boundInputs = new Set(); + const prerequisites = new Map>(); + for (const dependency of configuration.dependencies) { + const list = prerequisites.get(dependency.to) ?? []; + if (list.includes(dependency.from)) + return graphError("configure", `Duplicate dependency ${dependency.from}->${dependency.to}`); + list.push(dependency.from); + prerequisites.set(dependency.to, list); + const target = values.get(dependency.to); + const source = values.get(dependency.from); + if (source === undefined || target === undefined) + return graphError("configure", `Dependency references an unregistered instance`); + for (const binding of dependency.bindings ?? []) { + if (source.outputs[binding.output] === undefined) + return graphError("configure", `Unknown output ${dependency.from}.${binding.output}`); + if (!target.inputs.includes(binding.input)) + return graphError("configure", `Unknown input ${dependency.to}.${binding.input}`); + const inputKey = `${dependency.to}:${binding.input}`; + if (boundInputs.has(inputKey)) + return graphError("configure", `Duplicate binding for ${dependency.to}.${binding.input}`); + boundInputs.add(inputKey); + } + } + for (const member of configuration.members) { + if (!values.has(member.id)) + return graphError("configure", `Member references an unregistered instance ${member.id}`); + if (member.activation === "lazy" && !values.get(member.id)?.hasEndpoint) + return graphError("configure", `Lazy member ${member.id} has no public endpoint`); + if (member.idleMillis !== undefined && member.idleMillis < 0) + return graphError("configure", `Negative idle timeout for ${member.id}`); + } + const visiting = new Set(); + const visited = new Set(); + const visit = (id: string): OrchestratorError | undefined => { + if (visiting.has(id)) + return graphError("configure", "Composition dependencies contain a cycle"); + if (visited.has(id)) return; + visiting.add(id); + for (const prerequisite of prerequisites.get(id) ?? []) { + const failure = visit(prerequisite); + if (failure !== undefined) return failure; + } + visiting.delete(id); + visited.add(id); + }; + for (const id of values.keys()) { + const failure = visit(id); + if (failure !== undefined) return failure; + } + return { members, prerequisites }; +}; + +/** Every registered instance as the reducer sees it; a non-member runs as an eager standalone. */ +const lifecycleGraph = (ids: Iterable, configuration: CompositionConfig): LifecycleGraph => + makeGraph( + [...ids].map((id): ServiceSpec => { + const member = configuration.members.find((candidate) => candidate.id === id); + const prerequisites = configuration.dependencies + .filter((dependency) => dependency.to === id) + .map((dependency) => dependency.from); + return { + id, + activation: member?.activation ?? "eager", + prerequisites, + ...(member?.idleMillis === undefined ? {} : { idleMillis: member.idleMillis }), + }; + }), + ); + +const lifecycleOf = (state: LifecycleState, id: string): Status["lifecycle"] => { + const phase = state.services.get(id)?.phase; + switch (phase?._tag) { + case "Starting": + return "starting"; + case "Running": + return "running"; + case "Stopping": + return "stopping"; + default: + return "stopped"; + } +}; + +const statusOf = ( + state: LifecycleState, + id: string, + execution: ServiceObservation, +): Status => { + const service = state.services.get(id); + const phase = service?.phase; + const health = + phase?._tag === "Starting" + ? "starting" + : phase?._tag === "Running" + ? phase.ready + ? "healthy" + : service?.readinessFailure === undefined + ? "starting" + : "unhealthy" + : undefined; + return { + ...execution, + lifecycle: lifecycleOf(state, id), + health, + launchId: + phase === undefined || phase._tag === "Stopped" || phase._tag === "Failed" + ? undefined + : phase.generation, + wakeEnabled: service !== undefined && service.intent !== "stopped", + destroyPending: service?.destroy !== undefined, + }; +}; + +const rejectionFor = ( + commands: ReadonlyArray, + id: string, +): ServiceError | undefined => { + const rejected = commands.find( + (command) => command._tag === "RequestRejected" && command.id === id, + ); + return rejected?._tag === "RequestRejected" + ? new ServiceError({ operation: rejected.operation, message: rejected.message }) + : undefined; +}; + +const cleanupError = (id: string, cause: unknown) => + new ServiceError({ + operation: "stop", + message: + cause instanceof Error + ? `${id} cleanup failed: ${errorChainMessage(cause)}` + : `${id} cleanup failed`, + cause, + }); + +/** Builds an orchestrator whose lifecycle actor, timers and executions live in the current scope. */ +export const make = Effect.fn("Orchestrator.make")(function* ( + options: { + /** Refuses an explicit start, restart or storage operation, checked under the lifecycle gate. */ + readonly admit?: (operation: AdmittedOperation) => Effect.Effect; + } = {}, +): Effect.fn.Return, never, Scope.Scope> { const owner = yield* Scope.Scope; - const graphGate = yield* Semaphore.make(1); - const idleTimers = yield* FiberMap.make(); + const admit = options.admit ?? (() => Effect.void); + // The one lifecycle gate: every reducer step for this owner's graph is applied under it. + const gate = yield* Semaphore.make(1); + const lifecycle = yield* SubscriptionRef.make( + initialState(makeGraph([]), yield* Clock.currentTimeMillis), + ); const registry = yield* Ref.make>(new Map()); const composition = yield* Ref.make({ members: [], dependencies: [] }); - const activity = yield* Ref.make>(new Map()); - /** Tracks members with a wake in progress so only its initiating caller logs the transition. */ - const wakes = yield* Ref.make>(new Set()); - - const withGraph = Effect.fn("Orchestrator.withGraph")((effect: Effect.Effect) => - Effect.uninterruptibleMask((restore) => - restore(graphGate.take(1)).pipe( - Effect.andThen(effect.pipe(Effect.ensuring(graphGate.release(1)))), + const waiters = yield* Ref.make>(new Map()); + const nextWaiterId = yield* Ref.make(1); + const candidates = yield* Ref.make< + ReadonlyMap + >(new Map()); + const idleTimers = yield* FiberMap.make(); + // Nothing reduces events once the owner closes, so its pending waiters are released here. + yield* Scope.addFinalizer( + owner, + Ref.get(waiters).pipe( + Effect.flatMap((pending) => + Effect.forEach( + pending.values(), + ({ deferred }) => + Deferred.fail( + deferred, + new ServiceError({ operation: "lifecycle", message: "Lifecycle owner stopped" }), + ), + { discard: true }, + ), ), ), ); @@ -250,166 +402,80 @@ export const make = Effect.fn("Orchestrator.make")(function* < const node = Effect.fn("Orchestrator.node")(function* ( id: string, ): Effect.fn.Return { - yield* Effect.annotateCurrentSpan({ instance_id: id }); - const nodes = yield* Ref.get(registry); - const value = nodes.get(id); + const value = (yield* Ref.get(registry)).get(id); if (value === undefined) return yield* graphError("register", `Unknown instance ${id}`); return value; }); - const graph = ( - configuration: CompositionConfig, - values: ReadonlyMap, - ): Graph | OrchestratorError => { - const members = new Map(); - for (const member of configuration.members) { - if (members.has(member.id)) return graphError("configure", `Duplicate member ${member.id}`); - members.set(member.id, member); - } - const boundInputs = new Set(); - const prerequisites = new Map>(); - const dependents = new Map>(); - for (const dependency of configuration.dependencies) { - const list = prerequisites.get(dependency.to) ?? []; - if (list.includes(dependency.from)) - return graphError("configure", `Duplicate dependency ${dependency.from}->${dependency.to}`); - list.push(dependency.from); - prerequisites.set(dependency.to, list); - const reverse = dependents.get(dependency.from) ?? []; - reverse.push(dependency.to); - dependents.set(dependency.from, reverse); - const target = values.get(dependency.to); - const source = values.get(dependency.from); - if (source === undefined || target === undefined) - return graphError("configure", `Dependency references an unregistered instance`); - for (const binding of dependency.bindings ?? []) { - if (source.outputs[binding.output] === undefined) - return graphError("configure", `Unknown output ${dependency.from}.${binding.output}`); - if (!target.inputs.includes(binding.input)) - return graphError("configure", `Unknown input ${dependency.to}.${binding.input}`); - const inputKey = `${dependency.to}:${binding.input}`; - if (boundInputs.has(inputKey)) - return graphError("configure", `Duplicate binding for ${dependency.to}.${binding.input}`); - boundInputs.add(inputKey); - } - } - for (const member of configuration.members) { - if (!values.has(member.id)) - return graphError("configure", `Member references an unregistered instance ${member.id}`); - if (member.activation === "lazy" && !values.get(member.id)?.hasEndpoint) - return graphError("configure", `Lazy member ${member.id} has no public endpoint`); - if (member.idleMillis !== undefined && member.idleMillis < 0) - return graphError("configure", `Negative idle timeout for ${member.id}`); - } - const visiting = new Set(); - const visited = new Set(); - const visit = (id: string): OrchestratorError | undefined => { - if (visiting.has(id)) - return graphError("configure", "Composition dependencies contain a cycle"); - if (visited.has(id)) return; - visiting.add(id); - for (const prerequisite of prerequisites.get(id) ?? []) { - const failure = visit(prerequisite); - if (failure !== undefined) return failure; - } - visiting.delete(id); - visited.add(id); - }; - for (const id of values.keys()) { - const failure = visit(id); - if (failure !== undefined) return failure; - } - return { members, prerequisites, dependents }; - }; + /** + * Holds the gate for `effect`. The semaphore hands the permit over and installs its release in + * one protected step; only the wait for it keeps the caller's interruptibility. + */ + const withGate = (effect: Effect.Effect) => + gate.withPermit(Effect.uninterruptible(effect)); - const configurationGraph = Effect.fn("Orchestrator.configurationGraph")(function* ( - configuration: CompositionConfig, - ): Effect.fn.Return { - const values = yield* Ref.get(registry); - const result = graph(configuration, values); - if (result instanceof OrchestratorError) return yield* result; - return result; - }); + const fork = (effect: Effect.Effect) => + Effect.forkIn(effect, owner, { uninterruptible: false }).pipe(Effect.asVoid); - const operationBlocksDependents = (operation: ServiceAdmission) => - Match.value(operation).pipe( - Match.when("stop", () => true), - Match.when("restart", () => true), - Match.when("destroy", () => true), - Match.when("sleep", () => true), - Match.when("start", () => false), - Match.when("storage", () => false), - Match.when("arm", () => false), - Match.exhaustive, - ); - const dependentBlocks = (operation: ServiceAdmission, observation: CoreObservation) => - Match.value(operation).pipe( - Match.when("destroy", () => observation.registered), - Match.when("sleep", () => observation.lifecycle !== "stopped"), - Match.when("stop", () => isActive(observation)), - Match.when("restart", () => isActive(observation)), - Match.when("start", () => false), - Match.when("storage", () => false), - Match.when("arm", () => false), - Match.exhaustive, + const settleWaiter = (waiterId: number, result: Effect.Effect) => + Ref.get(waiters).pipe( + Effect.flatMap((current) => { + const waiter = current.get(waiterId); + return waiter === undefined ? Effect.void : Deferred.complete(waiter.deferred, result); + }), ); - const admissionFor = - (id: string) => (operation: ServiceAdmission, transition: Effect.Effect) => - withGraph( - Effect.gen(function* () { - const configured = yield* Ref.get(composition); - const structure = yield* configurationGraph(configured); - const dependents = structure.dependents.get(id) ?? []; - if (operationBlocksDependents(operation)) { - for (const dependentId of dependents) { - const dependent = yield* node(dependentId); - const observation = yield* dependent.core.get; - const blocked = dependentBlocks(operation, observation); - if (blocked) - return yield* serviceFailure( - "graph", - `Dependent ${dependentId} blocks ${operation} of ${id}`, - ); - } - } - if (operation === "sleep") { - const timeout = structure.members.get(id)?.idleMillis; - const current = (yield* Ref.get(activity)).get(id); - const now = yield* Clock.currentTimeMillis; - if ( - timeout === undefined || - current === undefined || - current.active > 0 || - now - current.lastActivity < timeout - ) - return yield* serviceFailure("graph", `Instance ${id} is not idle`); - } - if (operation === "start") { - for (const prerequisiteId of structure.prerequisites.get(id) ?? []) { - const prerequisite = yield* node(prerequisiteId); - const observation = yield* prerequisite.core.get; - if (observation.lifecycle !== "running" || observation.health !== "healthy") - return yield* serviceFailure( - "graph", - `Prerequisite ${prerequisiteId} is not healthy`, - ); - } - } - yield* transition; - }), - ).pipe( - Effect.mapError((error) => - error instanceof OrchestratorError ? serviceFailure("graph", error.message) : error, - ), - ); - const resolveInputs = Effect.fn("Orchestrator.resolveInputs")(function* ( + /** + * Applies events under a gate the caller already holds. Executions and timers are started before + * the new state is published, so whoever observes a transition also observes its armed timers; + * waiters are resolved last, once the state they were admitted against is visible. + */ + const applyLocked = Effect.fnUntraced(function* (events: ReadonlyArray) { + const now = yield* Clock.currentTimeMillis; + const before = yield* SubscriptionRef.get(lifecycle); + let after = before; + const commands: Array = []; + for (const event of events) { + const [next, emitted] = reduce(after, event, now); + after = next; + commands.push(...emitted); + } + const resolvesWaiter = (command: LifecycleCommand) => + command._tag === "AdmitConnection" || command._tag === "FailConnection"; + for (const command of commands) if (!resolvesWaiter(command)) yield* execute(command, after); + yield* SubscriptionRef.set(lifecycle, after); + for (const command of commands) if (resolvesWaiter(command)) yield* execute(command, after); + return { before, after, commands } satisfies Applied; + }); + + const dispatch = (events: ReadonlyArray) => withGate(applyLocked(events)); + + const report = (event: LifecycleEvent) => dispatch([event]).pipe(Effect.asVoid); + + /** Executes one `Stop` command; the service reports `Exited` or `StopFailed` itself. */ + const stopGeneration = Effect.fnUntraced(function* ( id: string, - configuration: CompositionConfig, - ): Effect.fn.Return, OrchestratorError | LifecycleError> { + generation: number, + state: LifecycleState, + ) { + const service = state.services.get(id); + const operation = + service?.intent === "stopped" ? "stop" : service?.relaunchForced ? "restart" : "sleep"; + const discard = service?.destroy !== undefined; + yield* fork( + node(id).pipe( + Effect.flatMap((entry) => entry.core.stop(generation, { operation, discard })), + Effect.withSpan("Lifecycle.stop", { + attributes: { instance_id: id, generation, operation }, + }), + ), + ); + }); + + const resolveInputs = Effect.fn("Orchestrator.resolveInputs")(function* (id: string) { + const configuration = yield* Ref.get(composition); const inputs: Record = {}; - const edges = configuration.dependencies.filter((dependency) => dependency.to === id); - for (const dependency of edges) { + for (const dependency of configuration.dependencies.filter(({ to }) => to === id)) { const source = yield* node(dependency.from); for (const binding of dependency.bindings ?? []) { const output = source.outputs[binding.output]; @@ -421,337 +487,612 @@ export const make = Effect.fn("Orchestrator.make")(function* < return inputs; }); - const snapshotPlan = Effect.fn("Orchestrator.snapshotPlan")(function* ( - id: string, - ): Effect.fn.Return { - return yield* withGraph( + const launch = Effect.fn("Lifecycle.launch")(function* (id: string, generation: number) { + yield* Effect.annotateCurrentSpan({ instance_id: id, generation }); + const candidate = yield* Ref.modify(candidates, (current) => { + const pending = current.get(id); + if (pending === undefined || generation < pending.minGeneration) return [undefined, current]; + const next = new Map(current); + next.delete(id); + return [pending.value, next]; + }); + const started = yield* Effect.gen(function* () { + const entry = yield* node(id); + const inputs = yield* resolveInputs(id); + yield* entry.launch(generation, inputs, candidate); + }).pipe(Effect.exit); + if (Exit.isFailure(started) && !Cause.hasInterruptsOnly(started.cause)) + yield* report( + LifecycleEvent.LaunchFailed({ id, generation, cause: Cause.squash(started.cause) }), + ); + }); + + // Interruptible so re-arming a key never waits out the timer it replaces. + const sleepThen = (delayMillis: number, event: LifecycleEvent) => + Effect.interruptible(Effect.sleep(`${delayMillis} millis`).pipe(Effect.andThen(report(event)))); + + /** Resolves waiters in place; every I/O command is forked onto the owner scope. */ + const execute = (command: LifecycleCommand, state: LifecycleState): Effect.Effect => { + switch (command._tag) { + case "AdmitConnection": + return settleWaiter(command.waiterId, Effect.void); + case "FailConnection": { + const cause = command.cause; + // A live session at failure time means the wait ended on readiness, not on the launch. + const operation = + state.services.get(command.id)?.phase._tag === "Running" + ? "readiness" + : cause instanceof ServiceError + ? cause.operation + : "wake"; + return settleWaiter( + command.waiterId, + Effect.fail( + new ServiceError({ + operation, + message: + cause instanceof Error + ? `${command.message}: ${errorChainMessage(cause)}` + : command.message, + ...(cause === undefined ? {} : { cause }), + }), + ), + ); + } + case "Launch": + return fork(launch(command.id, command.generation)); + case "Stop": + return stopGeneration(command.id, command.generation, state); + case "Reprobe": { + const reprobe = node(command.id).pipe( + Effect.flatMap((entry) => entry.core.reprobe(command.generation)), + Effect.ignore, + Effect.withSpan("Lifecycle.reprobe", { + attributes: { instance_id: command.id, generation: command.generation }, + }), + ); + if (command.delayMillis === 0) return fork(reprobe); + // Started at once so the spacing is already timed when the transition is published. + return Effect.forkIn( + Effect.sleep(`${command.delayMillis} millis`).pipe(Effect.andThen(reprobe)), + owner, + { startImmediately: true, uninterruptible: false }, + ).pipe(Effect.asVoid); + } + case "ArmIdleTimer": + return FiberMap.run( + idleTimers, + command.id, + sleepThen( + command.delayMillis, + LifecycleEvent.IdleElapsed({ + id: command.id, + generation: command.generation, + epoch: command.epoch, + }), + ), + { startImmediately: true }, + ).pipe(Effect.asVoid); + case "ArmCooldownTimer": + return Effect.forkIn( + sleepThen( + command.delayMillis, + LifecycleEvent.CooldownElapsed({ id: command.id, openUntil: command.openUntil }), + ), + owner, + { startImmediately: true, uninterruptible: false }, + ).pipe(Effect.asVoid); + case "ArmWaiterTimeout": + case "RequestRejected": + // The waiting or requesting caller reads these from the commands it dispatched. + return Effect.void; + } + }; + + interface Wait { + readonly id: string; + readonly requireReady: boolean; + } + + interface Admission { + /** Explicit requests dispatched atomically before the waiters; a rejection fails the call. */ + readonly requests?: ReadonlyArray>; + readonly waits: ReadonlyArray; + /** Traffic waits carry a lease and a wake budget; explicit waits carry neither. */ + readonly traffic: boolean; + /** Checks the owner's admission guard under the same gate step as the requests. */ + readonly operation?: AdmittedOperation; + /** Runs right after the dispatch, before waiting. */ + readonly onApplied?: (applied: Applied) => Effect.Effect; + /** Runs uninterruptibly once every wait is admitted, so a lease can't be lost to cancellation. */ + readonly onAdmitted?: Effect.Effect; + } + + /** + * Dispatches requests together with one waiter per wait, atomically, then awaits them all. Any + * failure, rejection or cancellation withdraws the remaining waiters and releases admitted leases. + */ + const awaitAll = Effect.fnUntraced(function* (admission: Admission) { + const { waits, traffic } = admission; + const openedAt = yield* Clock.currentTimeMillis; + const graph = (yield* SubscriptionRef.get(lifecycle)).graph; + const bounded = traffic ? waits[0] : undefined; + // Registration and its unconditional cleanup are installed together, before any wait. + return yield* Effect.uninterruptibleMask((restore) => Effect.gen(function* () { - const configuration = yield* Ref.get(composition); - const configured = yield* configurationGraph(configuration); - const structure: Graph = configured.members.has(id) - ? configured - : { - members: new Map([...configured.members, [id, { id, activation: "eager" as const }]]), - prerequisites: configured.prerequisites, - dependents: configured.dependents, - }; - const closure = new Set(); - const visit = (member: string) => { - if (closure.has(member)) return; - closure.add(member); - for (const prerequisite of structure.prerequisites.get(member) ?? []) visit(prerequisite); - }; - visit(id); - const revisions = new Map(); - for (const member of closure) { - const observation = yield* (yield* node(member)).core.get; - revisions.set(member, observation.intentRevision); - } - return { - order: topo([...closure], structure.prerequisites), - revisions, - structure, - configuration, - }; + const entries = yield* Effect.forEach(waits, (wait) => + Effect.all({ + waiterId: Ref.getAndUpdate(nextWaiterId, (value) => value + 1), + deferred: Deferred.make(), + }).pipe(Effect.map((created) => ({ ...wait, ...created }))), + ); + yield* Ref.update(waiters, (current) => { + const next = new Map(current); + for (const { waiterId, deferred } of entries) next.set(waiterId, { deferred }); + return next; + }); + const forget = Ref.update(waiters, (current) => { + const next = new Map(current); + for (const { waiterId } of entries) next.delete(waiterId); + return next; + }); + return yield* dispatchAndAwait(admission, entries, openedAt, graph, bounded, restore).pipe( + Effect.ensuring(forget), + ); }), ); }); - const prerequisiteGuard = Effect.fn("Orchestrator.prerequisiteGuard")(function* ( - plan: StartPlan, - id: string, - ): Effect.fn.Return { - if ((yield* Ref.get(composition)) !== plan.configuration) - return yield* serviceFailure("graph", "Composition changed before admission"); - const values = yield* Ref.get(registry); - for (const prerequisite of plan.structure.prerequisites.get(id) ?? []) { - const expected = plan.revisions.get(prerequisite); - const current = values.get(prerequisite); - if (expected === undefined || current === undefined) - return yield* serviceFailure("graph", `Missing prerequisite ${prerequisite}`); - const observation = yield* current.core.get; - if (observation.intentRevision !== expected) - return yield* serviceFailure( - "graph", - `Prerequisite ${prerequisite} changed before ${id} admission`, - ); + /** + * Dispatches the requests and waiters, then awaits them. Every failure, rejection or + * cancellation after the dispatch withdraws the waiters and releases granted leases. A traffic + * wait's budget covers the gate wait and the dispatch too. + */ + const dispatchAndAwait = Effect.fnUntraced(function* ( + admission: Admission, + entries: ReadonlyArray< + Wait & { readonly waiterId: number; readonly deferred: Deferred.Deferred } + >, + openedAt: number, + graph: LifecycleGraph, + bounded: Wait | undefined, + restore: (effect: Effect.Effect) => Effect.Effect, + ) { + const { requests = [], traffic } = admission; + const withdraw = Effect.gen(function* () { + yield* dispatch( + entries.map(({ id, waiterId }) => LifecycleEvent.WaiterCancelled({ id, waiterId })), + ); + if (!traffic) return; + for (const { id, deferred } of entries) { + if (!(yield* Deferred.isDone(deferred))) continue; + if (Exit.isSuccess(yield* Effect.exit(Deferred.await(deferred)))) + yield* dispatch([LifecycleEvent.ConnectionClosed({ id })]); + } + }); + // Set inside the gate, so a failure knows whether there is anything to withdraw. + const reached = yield* Ref.make(false); + const guarded = withGate( + (admission.operation === undefined ? Effect.void : admit(admission.operation)).pipe( + Effect.andThen(Ref.set(reached, true)), + Effect.andThen( + applyLocked([ + ...requests, + ...entries.map(({ id, waiterId, requireReady }) => + traffic + ? LifecycleEvent.ConnectionOpened({ id, waiterId, requireReady, openedAt }) + : LifecycleEvent.ReadinessAwaited({ id, waiterId, requireReady }), + ), + ]), + ), + ), + ); + const dispatched = yield* restore( + bounded === undefined + ? guarded + : guarded.pipe( + Effect.timeoutOrElse({ + duration: `${waiterBudgetMillis(graph, bounded.id)} millis`, + orElse: () => + Effect.fail( + new ServiceError({ + operation: "wake", + message: `${bounded.id} wake budget exceeded while waiting for admission`, + }), + ), + }), + ), + ).pipe(Effect.exit); + if (Exit.isFailure(dispatched)) { + if (yield* Ref.get(reached)) yield* withdraw; + return yield* Effect.failCause(dispatched.cause); } + const applied = dispatched.value; + for (const request of requests) { + const rejected = rejectionFor(applied.commands, request.id); + if (rejected === undefined) continue; + yield* withdraw; + return yield* rejected; + } + const awaitOne = ({ id, waiterId, deferred }: (typeof entries)[number]) => { + const timeout = applied.commands.find( + (command) => command._tag === "ArmWaiterTimeout" && command.waiterId === waiterId, + ); + if (timeout?._tag !== "ArmWaiterTimeout") return Deferred.await(deferred); + return Clock.currentTimeMillis.pipe( + Effect.flatMap((now) => + Deferred.await(deferred).pipe( + Effect.timeoutOption(`${Math.max(0, timeout.deadline - now)} millis`), + ), + ), + Effect.flatMap((admitted) => + Option.isSome(admitted) + ? Effect.void + : dispatch([LifecycleEvent.WaiterExpired({ id, waiterId })]).pipe( + Effect.andThen(Deferred.await(deferred)), + ), + ), + ); + }; + yield* admission.onApplied?.(applied) ?? Effect.void; + const outcome = yield* restore( + Effect.forEach(entries, awaitOne, { concurrency: "unbounded", discard: true }).pipe( + Effect.withSpan("Lifecycle.awaitAdmission", { + attributes: { + instance_ids: entries.map(({ id }) => id).join(","), + traffic, + waiters: entries.length, + }, + }), + ), + ).pipe(Effect.exit); + if (Exit.isFailure(outcome)) yield* withdraw; + else yield* admission.onAdmitted ?? Effect.void; + return yield* Effect.as(outcome, applied); }); - const startNode = Effect.fn("Orchestrator.startNode")(function* ( - id: string, - wake = false, - awaitTarget = false, - savedPlan?: StartPlan, - ): Effect.fn.Return { - yield* Effect.gen(function* () { - const plan = savedPlan ?? (yield* snapshotPlan(id)); - for (const member of plan.order) { - const instance = yield* node(member); - const inputs = yield* resolveInputs(member, plan.configuration); - const revision = plan.revisions.get(member); - if (revision === undefined) - return yield* graphError("start", `Missing revision for ${member}`); - yield* instance.startAt(revision, inputs, wake, prerequisiteGuard(plan, member)); - yield* instance.bind; - if (member !== id || awaitTarget) yield* instance.core.ready; - } + const prerequisitesOf = (state: LifecycleState, id: string): ReadonlyArray => + topo( + [...(state.graph.prerequisiteClosure.get(id) ?? [])], + new Map([...state.graph.services.values()].map((spec) => [spec.id, spec.prerequisites])), + ); + + /** The readiness waits that bound an explicit operation: every prerequisite, then the target. */ + const explicitWaits = Effect.fnUntraced(function* (id: string, targetReady: boolean) { + const state = yield* SubscriptionRef.get(lifecycle); + return [ + ...prerequisitesOf(state, id).map((prerequisite) => ({ + id: prerequisite, + requireReady: true, + })), + { id, requireReady: targetReady }, + ]; + }); + + const bindClosure = Effect.fn("Orchestrator.bindClosure")(function* (id: string) { + const state = yield* SubscriptionRef.get(lifecycle); + for (const member of [...prerequisitesOf(state, id), id]) yield* (yield* node(member)).bind; + }); + + /** Dispatches one explicit request, failing with the reducer's rejection if it refused it. */ + const request = (event: Extract) => + awaitAll({ requests: [event], waits: [], traffic: false }); + + const status = Effect.fn("Orchestrator.status")(function* (id: string) { + const entry = yield* node(id); + return statusOf(yield* SubscriptionRef.get(lifecycle), id, yield* entry.core.get); + }); + + const changes = (id: string): Stream.Stream => + Stream.unwrap( + node(id).pipe( + Effect.map((entry) => + Stream.zipLatestWith( + SubscriptionRef.changes(lifecycle), + entry.core.observation, + (state, execution) => statusOf(state, id, execution), + ), + ), + ), + ); + + const start = Effect.fn("Orchestrator.start")(function* (id: string) { + yield* Effect.annotateCurrentSpan({ instance_id: id }); + yield* node(id); + yield* bindClosure(id); + yield* awaitAll({ + requests: [LifecycleEvent.StartRequested({ id })], + waits: yield* explicitWaits(id, false), + traffic: false, + operation: "start", }); }); - const reschedulePrerequisites = Effect.fn("Orchestrator.reschedulePrerequisites")(function* ( - id: string, - ): Effect.fn.Return { - yield* Effect.gen(function* () { - const structure = yield* configurationGraph(yield* Ref.get(composition)); - for (const prerequisite of structure.prerequisites.get(id) ?? []) - yield* scheduleIdle(prerequisite); + const ready = Effect.fn("Orchestrator.ready")(function* (id: string) { + yield* Effect.annotateCurrentSpan({ instance_id: id }); + yield* node(id); + yield* awaitAll({ waits: yield* explicitWaits(id, true), traffic: false }); + }); + + /** Waits until `generation` is no longer stopping, failing while its cleanup is pending. */ + const awaitStopped = Effect.fnUntraced(function* (id: string, generation: number) { + const settled = yield* SubscriptionRef.changes(lifecycle).pipe( + Stream.map((state) => state.services.get(id)), + Stream.filter( + (service) => + service?.phase._tag !== "Stopping" || + service.phase.generation !== generation || + service.cleanupFailure !== undefined, + ), + Stream.runHead, + ); + const pending = Option.getOrUndefined(settled)?.cleanupFailure; + if (pending === undefined) return; + return yield* cleanupError(id, pending.cause); + }); + + /** Waits until a requested destroy holds the storage reservation, failing on pending cleanup. */ + const awaitDestroyReservation = Effect.fnUntraced(function* (id: string) { + const settled = yield* SubscriptionRef.changes(lifecycle).pipe( + Stream.map((state) => state.services.get(id)), + Stream.filter( + (service) => service?.destroy !== "requested" || service.cleanupFailure !== undefined, + ), + Stream.runHead, + ); + const service = Option.getOrUndefined(settled); + if (service?.destroy === "reserved") return; + if (service?.cleanupFailure !== undefined) + return yield* cleanupError(id, service.cleanupFailure.cause); + return yield* new ServiceError({ + operation: "destroy", + message: `${id} destroy was withdrawn`, }); }); - const scheduleIdle = Effect.fn("Orchestrator.scheduleIdle")(function* ( - id: string, - ): Effect.fn.Return { - yield* Effect.gen(function* () { - const configured = yield* Ref.get(composition); - const timeout = configured.members.find((member) => member.id === id)?.idleMillis; - if (timeout === undefined || timeout < 0) return; - const now = yield* Clock.currentTimeMillis; - const current = (yield* Ref.get(activity)).get(id); - if (current === undefined) return; - const delay = Math.max(0, timeout - (now - current.lastActivity)); - yield* FiberMap.run( - idleTimers, - id, - Effect.gen(function* () { - if (delay > 0) yield* Effect.sleep(`${delay} millis`); - const now = yield* Clock.currentTimeMillis; - const state = yield* withGraph( - Effect.gen(function* () { - const values = yield* Ref.get(activity); - const current = values.get(id); - return !( - current === undefined || - current.active > 0 || - now - current.lastActivity < timeout - ); - }), - ); - if (state) { - const result = yield* (yield* node(id)).core.sleep.pipe(Effect.exit); - if (Exit.isSuccess(result)) yield* reschedulePrerequisites(id); - } + const stop = Effect.fn("Orchestrator.stop")(function* (id: string) { + yield* Effect.annotateCurrentSpan({ instance_id: id }); + const entry = yield* node(id); + const applied = yield* request(LifecycleEvent.StopRequested({ id })); + const phase = applied.after.services.get(id)?.phase; + if (phase?._tag === "Stopping") + yield* awaitStopped(id, phase.generation).pipe( + Effect.withSpan("Lifecycle.awaitStopped", { + attributes: { instance_id: id, generation: phase.generation }, }), ); - }); + yield* entry.close; }); - const release = Effect.fn("Orchestrator.release")(function* ( - id: string, - ): Effect.fn.Return { - yield* Effect.gen(function* () { - const now = yield* Clock.currentTimeMillis; - yield* withGraph( + const restart = Effect.fn("Orchestrator.restart")(function* (id: string, config?: unknown) { + yield* Effect.annotateCurrentSpan({ instance_id: id }); + const entry = yield* node(id); + // A candidate is resolved and prepared before anything stops, so a bad one costs no downtime. + if (config !== undefined) yield* entry.prepare(yield* resolveInputs(id), config); + yield* bindClosure(id); + if (config !== undefined) { + // Any launch from the restart's own generation onward applies the candidate. + const minGeneration = (yield* SubscriptionRef.get(lifecycle)).generationCounters.get(id) ?? 1; + yield* Ref.update(candidates, (current) => + new Map(current).set(id, { minGeneration, value: config }), + ); + } + // Once admitted, the candidate belongs to the next launch even if this caller's wait fails. + const admitted = yield* Ref.make(false); + yield* awaitAll({ + requests: [LifecycleEvent.RestartRequested({ id })], + waits: yield* explicitWaits(id, false), + traffic: false, + operation: "restart", + onApplied: () => Ref.set(admitted, true), + }).pipe( + Effect.tapError(() => + Ref.get(admitted).pipe( + Effect.flatMap((wasAdmitted) => + wasAdmitted + ? Effect.void + : Ref.update(candidates, (current) => { + const next = new Map(current); + next.delete(id); + return next; + }), + ), + ), + ), + ); + }); + + const storage = (id: string, operation: Effect.Effect) => + Effect.gen(function* () { + const entry = yield* node(id); + return yield* Effect.acquireUseRelease( + awaitAll({ + requests: [LifecycleEvent.StorageReserved({ id })], + waits: [], + traffic: false, + operation: "storage", + }), + () => entry.core.storage(operation), + () => dispatch([LifecycleEvent.StorageReleased({ id })]), + ); + }).pipe(Effect.withSpan("Orchestrator.storage", { attributes: { instance_id: id } })); + + const unregister = (id: string) => + withGate( + Effect.gen(function* () { + const values = new Map(yield* Ref.get(registry)); + values.delete(id); + yield* Ref.set(registry, values); + const configured = yield* Ref.get(composition); + const next: CompositionConfig = { + members: configured.members.filter((member) => member.id !== id), + dependencies: configured.dependencies.filter( + (dependency) => dependency.from !== id && dependency.to !== id, + ), + }; + yield* Ref.set(composition, next); + yield* applyLocked([ + LifecycleEvent.DestroyReleased({ id }), + LifecycleEvent.GraphUpdated({ graph: lifecycleGraph(values.keys(), next) }), + ]); + }), + ); + + const destroy = Effect.fn("Orchestrator.destroy")(function* (id: string) { + yield* Effect.annotateCurrentSpan({ instance_id: id }); + const entry = yield* node(id); + const configured = yield* Ref.get(composition); + const dependent = configured.dependencies.find((dependency) => dependency.from === id); + if (dependent !== undefined) + return yield* new ServiceError({ + operation: "graph", + message: `Dependent ${dependent.to} blocks destroy of ${id}`, + }); + // Any exit short of unregistering gives the destroy up, so the service is usable again. + yield* Effect.uninterruptibleMask((restore) => + restore( Effect.gen(function* () { - const values = yield* Ref.get(activity); - const current = values.get(id); - if (current === undefined) return; - yield* Ref.set( - activity, - new Map(values).set(id, { - active: Math.max(0, current.active - 1), - lastActivity: now, + yield* request(LifecycleEvent.DestroyRequested({ id })); + yield* awaitDestroyReservation(id).pipe( + Effect.withSpan("Lifecycle.awaitDestroyReservation", { + attributes: { instance_id: id }, }), ); + yield* entry.close; + yield* entry.core.removeData; + yield* entry.close; }), - ); - yield* scheduleIdle(id); - }); + ).pipe( + Effect.onExit((exit) => + Exit.isSuccess(exit) + ? unregister(id) + : dispatch([LifecycleEvent.DestroyReleased({ id })]), + ), + ), + ); }); - const settle = Effect.fn("Orchestrator.settle")(function* ( + const settleOutcomes = Effect.fn("Orchestrator.settleOutcomes")(function* ( operation: "start" | "stop" | "destroy", ids: ReadonlyArray, - action: (id: string) => Effect.Effect, + action: (id: string) => Effect.Effect, concurrency: 1 | "unbounded" = 1, - ): Effect.fn.Return, OrchestratorError | LifecycleError> { - return yield* Effect.gen(function* () { - const outcomes = yield* Effect.forEach( - ids, - (id) => - action(id).pipe( - Effect.exit, - Effect.map((result) => ({ id, result })), - ), - { concurrency }, - ); - if (outcomes.some(({ result }) => Exit.isFailure(result))) - return yield* new OrchestratorError({ - operation, - message: `Composition ${operation} had failures`, - outcomes, - }); - return yield* Effect.forEach(ids, (id) => - node(id).pipe(Effect.flatMap((instance) => instance.core.get)), - ); - }); + ) { + const outcomes = yield* Effect.forEach( + ids, + (id) => + action(id).pipe( + Effect.exit, + Effect.map((result) => ({ id, result })), + ), + { concurrency }, + ); + if (outcomes.some(({ result }) => Exit.isFailure(result))) + return yield* new OrchestratorError({ + operation, + message: `Composition ${operation} had failures`, + outcomes, + }); + return yield* Effect.forEach(ids, status); }); - const startComposition = Effect.fn("Orchestrator.startComposition")( - function* (): Effect.fn.Return< - ReadonlyArray, - OrchestratorError | LifecycleError - > { - const configured = yield* Ref.get(composition); - const structure = yield* configurationGraph(configured); - const order = topo( - configured.members.map((member) => member.id), - structure.prerequisites, - ); - const eager = new Set( - topo( - [ - ...configured.members - .filter((member) => member.activation === "eager") - .map((member) => member.id), - ...order.filter((id) => !structure.members.has(id)), - ], - structure.prerequisites, + const startComposition = Effect.fn("Orchestrator.startComposition")(function* () { + const configured = yield* Ref.get(composition); + const structure = validateGraph(configured, yield* Ref.get(registry)); + if (structure instanceof OrchestratorError) return yield* structure; + const order = topo( + configured.members.map((member) => member.id), + structure.prerequisites, + ); + const eager = new Set(order.filter((id) => structure.members.get(id)?.activation !== "lazy")); + const completions = new Map>(); + for (const id of order) completions.set(id, yield* Deferred.make()); + const bound = new Map>(); + for (const id of order) + bound.set( + id, + yield* node(id).pipe( + Effect.flatMap((entry) => entry.bind), + Effect.exit, ), ); - const plans = new Map(); - const bound = new Map>(); - const completions = new Map< - string, - Deferred.Deferred> - >(); - for (const id of order) plans.set(id, yield* snapshotPlan(id)); - for (const id of order) - completions.set( - id, - yield* Deferred.make>(), - ); - for (const id of order) - bound.set( - id, - yield* node(id).pipe( - Effect.flatMap((instance) => instance.bind), - Effect.exit, - ), - ); - return yield* settle( - "start", - order, - (id) => { - const completion = completions.get(id); - if (completion === undefined) - return Effect.fail(graphError("start", `Missing completion for ${id}`)); - const action = Effect.gen(function* () { - const prerequisites = structure.prerequisites.get(id) ?? []; - const prerequisiteOutcomes = yield* Effect.forEach(prerequisites, (prerequisite) => { - const dependency = completions.get(prerequisite); - return dependency === undefined - ? Effect.fail(graphError("start", `Missing completion for ${prerequisite}`)) - : Deferred.await(dependency).pipe( - Effect.map((result) => ({ id: prerequisite, result })), - ); + return yield* settleOutcomes( + "start", + order, + (id) => + Effect.gen(function* () { + const failedPrerequisites: Array = []; + for (const prerequisite of structure.prerequisites.get(id) ?? []) { + const completion = completions.get(prerequisite); + if (completion !== undefined && !(yield* Deferred.await(completion))) + failedPrerequisites.push(prerequisite); + } + const binding = bound.get(id); + if (binding !== undefined) yield* binding; + if (!eager.has(id)) { + yield* awaitAll({ + requests: [LifecycleEvent.ArmRequested({ id })], + waits: [], + traffic: false, + operation: "start", }); - const binding = bound.get(id); - if (binding !== undefined) yield* binding; - const instance = yield* node(id); - const plan = plans.get(id); - const revision = plan?.revisions.get(id); - if (plan === undefined || revision === undefined) - return yield* graphError("start", `Missing plan for ${id}`); - const policy = structure.members.get(id); - if ( - policy?.activation === "lazy" || - (policy?.idleMillis !== undefined && instance.hasEndpoint) - ) - yield* instance.core.armAt(revision, prerequisiteGuard(plan, id)); - if (eager.has(id)) { - const failedPrerequisites = prerequisiteOutcomes - .filter(({ result }) => Exit.isFailure(result)) - .map(({ id: prerequisite }) => prerequisite) - .toSorted((left, right) => order.indexOf(left) - order.indexOf(right)); - if (failedPrerequisites.length > 0) - return yield* graphError( - "start", - `Blocked by prerequisites: ${failedPrerequisites.join(", ")}`, - ); - const inputs = yield* resolveInputs(id, configured); - yield* instance.startAt(revision, inputs, false, prerequisiteGuard(plan, id)); - yield* instance.bind; - yield* instance.core.ready; - } + return; + } + if (failedPrerequisites.length > 0) + return yield* graphError( + "start", + `Blocked by prerequisites: ${failedPrerequisites + .toSorted((left, right) => order.indexOf(left) - order.indexOf(right)) + .join(", ")}`, + ); + yield* awaitAll({ + requests: [LifecycleEvent.StartRequested({ id })], + waits: yield* explicitWaits(id, true), + traffic: false, + operation: "start", }); - return action.pipe( - Effect.onExit((result) => Deferred.succeed(completion, result).pipe(Effect.asVoid)), - ); - }, - "unbounded", - ); - }, - ); + }).pipe( + Effect.onExit((exit) => { + const completion = completions.get(id); + return completion === undefined + ? Effect.void + : Deferred.succeed(completion, Exit.isSuccess(exit)).pipe(Effect.asVoid); + }), + ), + "unbounded", + ); + }); const stopIds = Effect.fn("Orchestrator.stopIds")(function* (ids: ReadonlyArray) { - const structure = yield* configurationGraph(yield* Ref.get(composition)); + const configured = yield* Ref.get(composition); + const structure = validateGraph(configured, yield* Ref.get(registry)); + if (structure instanceof OrchestratorError) return yield* structure; const order = topo(ids, structure.prerequisites) .filter((id) => ids.includes(id)) .toReversed(); - return yield* settle("stop", order, (id) => - Effect.gen(function* () { - const instance = yield* node(id); - yield* instance.core.stop; - yield* instance.close; - yield* reschedulePrerequisites(id); - }), - ); - }); - - const destroyNode = Effect.fn("Orchestrator.destroyNode")(function* (id: string) { - const instance = yield* node(id); - yield* instance.core.destroy; - yield* instance.close; - yield* withGraph( - Effect.gen(function* () { - const values = yield* Ref.get(registry); - const next = new Map(values); - next.delete(id); - yield* Ref.set(registry, next); - const activities = new Map(yield* Ref.get(activity)); - activities.delete(id); - yield* Ref.set(activity, activities); - const configured = yield* Ref.get(composition); - yield* Ref.set(composition, { - members: configured.members.filter((member) => member.id !== id), - dependencies: configured.dependencies.filter( - (dependency) => dependency.from !== id && dependency.to !== id, - ), - }); - }), - ); + return yield* settleOutcomes("stop", order, stop); }); const stopComposition = Effect.fn("Orchestrator.stopComposition")(function* () { - const configured = yield* Ref.get(composition); - return yield* stopIds(configured.members.map((member) => member.id)); + return yield* stopIds((yield* Ref.get(composition)).members.map((member) => member.id)); }); + const restartComposition = Effect.fn("Orchestrator.restartComposition")(function* () { - const configured = yield* Ref.get(composition); - yield* stopIds(configured.members.map((member) => member.id)); + yield* stopIds((yield* Ref.get(composition)).members.map((member) => member.id)); return yield* startComposition(); }); + const stopNamespace = Effect.fn("Orchestrator.stopNamespace")(function* () { - const values = yield* Ref.get(registry); - yield* stopIds([...values.keys()]); + yield* stopIds([...(yield* Ref.get(registry)).keys()]); }); + const destroyNamespace = Effect.fn("Orchestrator.destroyNamespace")(function* () { const values = yield* Ref.get(registry); - const configured = yield* configurationGraph(yield* Ref.get(composition)); - const order = topo([...values.keys()], configured.prerequisites).toReversed(); + const structure = validateGraph(yield* Ref.get(composition), values); + if (structure instanceof OrchestratorError) return yield* structure; + const order = topo([...values.keys()], structure.prerequisites).toReversed(); const outcomes = yield* Effect.forEach(order, (id) => - destroyNode(id).pipe( + destroy(id).pipe( Effect.exit, Effect.map((result) => ({ id, result })), ), @@ -764,215 +1105,138 @@ export const make = Effect.fn("Orchestrator.make")(function* < }); }); + const acquire = Effect.fn("Orchestrator.acquire")(function* ( + id: string, + awaitReady = true, + trigger?: string, + ) { + yield* Effect.annotateCurrentSpan({ instance_id: id, await_ready: awaitReady }); + const entry = yield* node(id); + const scope = yield* Scope.Scope; + const initiated = yield* Ref.make(false); + // The caller that takes a sleeping or failed service out of rest names the wake once. + const announce = (applied: Applied) => { + const before = applied.before.services.get(id); + const after = applied.after.services.get(id); + const wakes = + before !== undefined && + (before.phase._tag === "Stopped" || before.phase._tag === "Failed") && + before.waiters.size === 0 && + (after?.waiters.size ?? 0) > 0; + return wakes + ? Ref.set(initiated, true).pipe( + Effect.andThen( + Effect.logInfo( + `Waking ${entry.service} ${id}${trigger === undefined ? "" : ` (${trigger})`}`, + ), + ), + ) + : Effect.void; + }; + yield* awaitAll({ + waits: [{ id, requireReady: awaitReady }], + traffic: true, + onApplied: announce, + onAdmitted: Scope.addFinalizer(scope, dispatch([LifecycleEvent.ConnectionClosed({ id })])), + }).pipe( + Effect.tapError((error) => + Ref.get(initiated).pipe( + Effect.flatMap((initiator) => + initiator + ? Effect.logError( + error.operation === "readiness" + ? `${entry.service} ${id} failed to become ready` + : `${entry.service} ${id} failed to wake`, + error, + ) + : Effect.void, + ), + ), + ), + ); + if (yield* Ref.get(initiated)) + yield* Effect.logInfo( + awaitReady + ? `${entry.service} ${id} is ready` + : `${entry.service} ${id} started (readiness not awaited)`, + ); + }); + const orchestrator: Interface = { - admissionFor, + report, register: Effect.fn("Orchestrator.register")(function* (instance) { yield* Effect.annotateCurrentSpan({ instance_id: instance.id }); - return yield* withGraph( + yield* withGate( Effect.gen(function* () { const values = yield* Ref.get(registry); if (values.has(instance.id)) return yield* graphError("register", `Duplicate instance ${instance.id}`); - yield* Ref.set(registry, new Map(values).set(instance.id, instance)); - const seenExit = yield* Ref.make(undefined); - const seenHealthy = yield* Ref.make(undefined); - yield* Effect.forkIn( - instance.core.observation.pipe( - Stream.takeUntil((observation) => !observation.registered), - Stream.runForEach((observation) => - Effect.gen(function* () { - if ( - observation.lifecycle === "running" && - observation.health === "healthy" && - observation.wakeEnabled && - observation.launchId !== (yield* Ref.get(seenHealthy)) - ) { - yield* Ref.set(seenHealthy, observation.launchId); - const now = yield* Clock.currentTimeMillis; - yield* withGraph( - Ref.update(activity, (values) => - new Map(values).set(instance.id, { - active: values.get(instance.id)?.active ?? 0, - lastActivity: now, - }), - ), - ); - yield* scheduleIdle(instance.id); - } - if (observation.exit === undefined) return; - if (observation.lifecycle === "stopped") { - yield* reschedulePrerequisites(instance.id); - return; - } - if (observation.wakeEnabled) return; - if ((yield* Ref.get(seenExit)) === observation.exit) return; - yield* Ref.set(seenExit, observation.exit); - yield* instance.close.pipe( - Effect.tapError((cause) => Effect.logError("Instance close failed", cause)), - Effect.ignore, - ); - }), - ), - ), - owner, - ); + const next = new Map(values).set(instance.id, instance); + yield* Ref.set(registry, next); + yield* applyLocked([ + LifecycleEvent.GraphUpdated({ + graph: lifecycleGraph(next.keys(), yield* Ref.get(composition)), + }), + ]); }), ); }), - get: Effect.fn("Orchestrator.get")((id) => node(id)), + get: (id) => node(id), + status, + changes, composition: Ref.get(composition), configure: (configuration: CompositionConfig, persist?: Effect.Effect) => - withGraph( + withGate( Effect.gen(function* () { const previous = yield* Ref.get(composition); - yield* configurationGraph(configuration); + const values = yield* Ref.get(registry); + const structure = validateGraph(configuration, values); + if (structure instanceof OrchestratorError) return yield* structure; const affected = new Set([ ...previous.members.map((member) => member.id), ...configuration.members.map((member) => member.id), + ...[...previous.dependencies, ...configuration.dependencies].flatMap((dependency) => [ + dependency.from, + dependency.to, + ]), ]); - for (const dependency of [...previous.dependencies, ...configuration.dependencies]) { - affected.add(dependency.from); - affected.add(dependency.to); - } + const state = yield* SubscriptionRef.get(lifecycle); for (const affectedId of affected) { - const instance = yield* node(affectedId); - const observation = yield* instance.core.get; - if (observation.lifecycle !== "stopped" || observation.wakeEnabled) + const service = state.services.get(affectedId); + if ( + service !== undefined && + (service.intent !== "stopped" || lifecycleOf(state, affectedId) !== "stopped") + ) return yield* graphError( "configure", `Instance ${affectedId} must be stopped and wake-disabled`, ); } + const update = LifecycleEvent.GraphUpdated({ + graph: lifecycleGraph(values.keys(), configuration), + }); + // The reducer is pure, so its verdict is known before anything is persisted. + const [, verdict] = reduce(state, update, yield* Clock.currentTimeMillis); + const rejected = verdict.find((command) => command._tag === "RequestRejected"); + if (rejected?._tag === "RequestRejected") + return yield* graphError("configure", rejected.message); if (persist !== undefined) yield* persist; yield* Ref.set(composition, configuration); + yield* applyLocked([update]); }), ).pipe(Effect.withSpan("Orchestrator.configure")), - start: Effect.fn("Orchestrator.start")((id) => - Effect.gen(function* () { - yield* Effect.annotateCurrentSpan({ instance_id: id }); - const plan = yield* snapshotPlan(id); - for (const member of plan.order) yield* (yield* node(member)).bind; - yield* startNode(id, false, false, plan); - }), - ), - stop: Effect.fn("Orchestrator.stop")((id) => - Effect.gen(function* () { - yield* Effect.annotateCurrentSpan({ instance_id: id }); - const instance = yield* node(id); - yield* instance.core.stop; - yield* instance.close; - yield* reschedulePrerequisites(id); - }), - ), - restart: Effect.fn("Orchestrator.restart")((id, config) => - Effect.gen(function* () { - yield* Effect.annotateCurrentSpan({ instance_id: id }); - const plan = yield* snapshotPlan(id); - for (const member of plan.order) yield* (yield* node(member)).bind; - for (const member of plan.order.slice(0, -1)) { - const instance = yield* node(member); - const inputs = yield* resolveInputs(member, plan.configuration); - const revision = plan.revisions.get(member); - if (revision === undefined) - return yield* graphError("restart", `Missing revision for ${member}`); - yield* instance.startAt(revision, inputs, false, prerequisiteGuard(plan, member)); - yield* instance.bind; - yield* instance.core.ready; - } - const instance = yield* node(id); - const inputs = yield* resolveInputs(id, plan.configuration); - const revision = plan.revisions.get(id); - if (revision === undefined) - return yield* graphError("restart", `Missing revision for ${id}`); - yield* instance.restart(revision, inputs, config, prerequisiteGuard(plan, id)); - yield* instance.bind; - }), - ), - destroy: Effect.fn("Orchestrator.destroy")((id) => - Effect.annotateCurrentSpan({ instance_id: id }).pipe(Effect.andThen(destroyNode(id))), - ), + start, + ready, + stop, + restart, + destroy, + storage, startComposition: startComposition(), stopComposition: stopComposition(), restartComposition: restartComposition(), stopNamespace: stopNamespace(), destroyNamespace: destroyNamespace(), - acquire: Effect.fn("Orchestrator.acquire")((id, awaitReady = true, trigger) => - Effect.gen(function* () { - yield* Effect.annotateCurrentSpan({ instance_id: id }); - const instance = yield* node(id); - const scope = yield* Scope.Scope; - const now = yield* Clock.currentTimeMillis; - const { wake, initiator } = yield* withGraph( - Effect.gen(function* () { - const observation = yield* instance.core.get; - if (observation.lifecycle === "stopped" && !observation.wakeEnabled) - return yield* serviceFailure("proxy", `Instance ${id} is explicitly stopped`); - const values = yield* Ref.get(activity); - yield* Ref.set( - activity, - new Map(values).set(id, { - active: (values.get(id)?.active ?? 0) + 1, - lastActivity: now, - }), - ); - yield* Scope.addFinalizer( - scope, - release(id).pipe( - Effect.tapError((cause) => Effect.logError("Instance release failed", cause)), - Effect.ignore, - ), - ); - // A "starting" observer is always joining an in-flight start (a wake or an - // eager/explicit start), never its initiator, regardless of the marker below. - if (observation.lifecycle === "running" || observation.lifecycle === "starting") - return { wake: observation.lifecycle === "starting", initiator: false } as const; - const inFlight = yield* Ref.get(wakes); - const initiator = !inFlight.has(id); - if (initiator) yield* Ref.set(wakes, new Set(inFlight).add(id)); - return { wake: true, initiator } as const; - }), - ); - if (wake) { - if (initiator) - yield* Effect.logInfo( - `Waking ${instance.service} ${id}${trigger === undefined ? "" : ` (${trigger})`}`, - ); - yield* Effect.gen(function* () { - const plan = yield* snapshotPlan(id); - for (const member of plan.order) yield* (yield* node(member)).bind; - yield* startNode(id, true, false, plan).pipe( - Effect.tapError((cause) => - initiator - ? Effect.logError(`${instance.service} ${id} failed to wake`, cause) - : Effect.void, - ), - ); - }).pipe( - Effect.ensuring( - initiator - ? Ref.update(wakes, (ids) => { - const next = new Set(ids); - next.delete(id); - return next; - }) - : Effect.void, - ), - ); - if (!awaitReady && initiator) - yield* Effect.logInfo(`${instance.service} ${id} started (readiness not awaited)`); - } - if (awaitReady) { - yield* instance.core.ready.pipe( - Effect.tapError((cause) => - wake && initiator - ? Effect.logError(`${instance.service} ${id} failed to become ready`, cause) - : Effect.void, - ), - ); - if (wake && initiator) yield* Effect.logInfo(`${instance.service} ${id} is ready`); - } - }), - ), + acquire, }; return orchestrator; }); diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index b3bf0d688e..e2eca10a57 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -27,10 +27,8 @@ import type { CompositionConfig } from "./Orchestrator.ts"; import { makeService, ServiceError, - type ServiceAdmission, type ServiceInstance, type ServiceInstanceContext, - type ServiceObservation, } from "./Service.ts"; import { ProxyError } from "./Proxy.ts"; import { @@ -161,7 +159,7 @@ export const sweepContainers = Effect.fn("Owner.sweepContainers")(function* ( type NamespaceError = | Orchestrator.OrchestratorError - | Orchestrator.LifecycleError + | ServiceError | Network.NetworkError | StackNamespace.NamespaceError | Effect.Error>; @@ -242,8 +240,6 @@ const withoutInstance = (current: SavedStack, id: string): SavedStack => }, }); -const drainingBlocks: ReadonlyArray = ["start", "arm", "restart", "storage"]; - const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { const services = yield* Effect.context< | FileSystem.FileSystem @@ -269,10 +265,17 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { }), ), ); - const orchestrator = yield* Orchestrator.make(); + const draining = yield* Ref.make(false); + const rejectWhileDraining = Ref.get(draining).pipe( + Effect.flatMap((isDraining) => + isDraining + ? Effect.fail(new ServiceError({ operation: "draining", message: "Owner is draining" })) + : Effect.void, + ), + ); + const orchestrator = yield* Orchestrator.make({ admit: () => rejectWhileDraining }); const helpers = yield* makeDockerHelperRegistry(yield* crypto.randomUUIDv4); const definitionGate = yield* Semaphore.make(1); - const draining = yield* Ref.make(false); const { id: stackId, runtime } = options.saved; const project = projectSegmentFor(options.saved.identity, path); const routeKeys = { @@ -282,32 +285,41 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { serviceRoleKey: options.saved.credentials?.serviceRoleKey ?? "", }; - const rejectWhileDraining = Ref.get(draining).pipe( - Effect.flatMap((isDraining) => - isDraining - ? Effect.fail(new ServiceError({ operation: "draining", message: "Owner is draining" })) - : Effect.void, - ), - ); - - // Mask only the permit handoff; admitted definition changes belong to the owner scope. A change - // arriving while draining fails before waiting behind the shutdown that holds the permit. + // Admitted definition changes run in the owner scope, so a caller cancelled after admission + // detaches from them; one still queued for the permit is withdrawn. A change arriving while + // draining fails before waiting behind the shutdown that holds the permit. const definitionChange = (work: Effect.Effect) => - Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - yield* rejectWhileDraining; - yield* restore(definitionGate.take(1)); - const fiber = yield* Effect.forkIn( - rejectWhileDraining.pipe( - Effect.andThen(work), - Effect.ensuring(definitionGate.release(1)), - ), - ownerScope, - { uninterruptible: false }, - ); - return yield* restore(Fiber.join(fiber)); - }), - ); + Effect.gen(function* () { + yield* rejectWhileDraining; + const claim = yield* Ref.make<"queued" | "admitted" | "withdrawn">("queued"); + const settle = (to: "admitted" | "withdrawn") => + Ref.modify(claim, (current) => (current === "queued" ? [true, to] : [false, current])); + return yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const worker = yield* Effect.forkIn( + definitionGate.withPermit( + settle("admitted").pipe( + Effect.flatMap((admitted) => + admitted + ? rejectWhileDraining.pipe(Effect.andThen(work), Effect.map(Option.some)) + : Effect.succeed(Option.none()), + ), + ), + ), + ownerScope, + { uninterruptible: false }, + ); + const outcome = yield* restore(Fiber.join(worker)).pipe( + Effect.onInterrupt(() => + settle("withdrawn").pipe( + Effect.flatMap((withdrawn) => (withdrawn ? Fiber.interrupt(worker) : Effect.void)), + ), + ), + ); + return Option.isSome(outcome) ? outcome.value : yield* Effect.interrupt; + }), + ); + }); const readSaved = options.state.read(stackId).pipe( Effect.flatMap((saved) => @@ -346,8 +358,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ...configuration.dependencies.flatMap(({ from, to }) => [from, to]), ]), (id) => - orchestrator.get(id).pipe( - Effect.flatMap((entry) => entry.core.get), + orchestrator.status(id).pipe( Effect.flatMap(({ lifecycle, wakeEnabled }) => lifecycle === "stopped" && !wakeEnabled ? Effect.void @@ -451,23 +462,21 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ), ), }, - { - id, - config: initial, - coordinate: (operation, transition) => - (drainingBlocks.includes(operation) ? rejectWhileDraining : Effect.void).pipe( - Effect.andThen(orchestrator.admissionFor(id)(operation, transition)), - ), - }, + { id, config: initial, report: orchestrator.report }, ).pipe(Effect.provideService(Scope.Scope, ownerScope)); - const enabled = core.get.pipe( + // Listeners stay open while the service runs or demand can still wake it. + const enabled = orchestrator.status(id).pipe( Effect.map( - (observation) => - observation.registered && - observation.lifecycle !== "stopped" && - !(observation.exit !== undefined && !observation.wakeEnabled), + (status) => status.registered && (status.wakeEnabled || status.lifecycle !== "stopped"), ), + Effect.orElseSucceed(() => false), ); + const configFor = (inputs: Record, candidate: unknown) => + Ref.get(creation).pipe( + Effect.flatMap((current) => restartCreation(current, candidate)), + Effect.flatMap((next) => mergeInputs(next, inputs)), + Effect.flatMap(requireInputs), + ); const endpoints = Object.fromEntries( endpointNames(initial).map((name) => { const shared = sharedRoutes(initial, name, routeKeys); @@ -503,19 +512,12 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { creation, namespace, cleanupResources, - startAt: (revision, inputs, wake, guard) => - Ref.get(creation).pipe( - Effect.flatMap((current) => mergeInputs(current, inputs)), - Effect.flatMap(requireInputs), - Effect.flatMap((candidate) => core.startAt(revision, candidate, wake, guard)), - ), - restart: (revision, inputs, candidate, guard) => - Ref.get(creation).pipe( - Effect.flatMap((current) => restartCreation(current, candidate)), - Effect.flatMap((next) => mergeInputs(next, inputs)), - Effect.flatMap(requireInputs), - Effect.flatMap((next) => core.restart(next, revision, guard)), + launch: (generation, inputs, candidate) => + configFor(inputs, candidate).pipe( + Effect.flatMap((config) => core.launch(generation, config)), ), + prepare: (inputs, candidate) => + configFor(inputs, candidate).pipe(Effect.flatMap((config) => core.prepare(config))), bind: namespace.bind.pipe(Effect.mapError(serviceError("bind"))), close: namespace.close.pipe(Effect.mapError(serviceError("close"))), hasEndpoint: endpointNames(initial).length > 0, @@ -550,7 +552,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { type ComposeError = | Effect.Error> - | Orchestrator.LifecycleError + | ServiceError | Network.NetworkError; const configure = (configuration: CompositionConfig) => @@ -584,7 +586,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { Effect.flatMap((entry) => Ref.get(entry.creation)), Effect.map((creation) => ({ id, creation })), ), - status: (id) => orchestrator.get(id).pipe(Effect.flatMap((entry) => entry.core.get)), + status: orchestrator.status, create: addInstance, destroy: orchestrator.destroy, bind: (id) => orchestrator.get(id).pipe(Effect.flatMap((entry) => entry.bind)), @@ -647,18 +649,14 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { return yield* orchestrator.restart(id, next); }); - const observe = (entry: Entry, value: ServiceObservation) => + const observe = (entry: Entry, value: Orchestrator.Status) => Effect.all({ config: Ref.get(entry.creation), endpoints: entry.namespace.bindings }).pipe( Effect.map((current) => ({ ...value, ...current })), ); const observation = (id: string) => - orchestrator - .get(id) - .pipe( - Effect.flatMap((entry) => - entry.core.get.pipe(Effect.flatMap((value) => observe(entry, value))), - ), - ); + Effect.all([orchestrator.get(id), orchestrator.status(id)]).pipe( + Effect.flatMap(([entry, value]) => observe(entry, value)), + ); const observeAll = (values: ReadonlyArray<{ readonly id: string }>) => Effect.forEach(values, ({ id }) => observation(id)); @@ -677,7 +675,8 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { operation: "storage", message: "Snapshots and data reset are only supported for databases", }); - return yield* entry.core.storage( + return yield* orchestrator.storage( + id, Effect.acquireUseRelease( Scope.make("parallel"), (scope) => @@ -707,11 +706,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { }), ).pipe(rpcError("createService")), startService: ({ id }) => orchestrator.start(id).pipe(rpcError("startService")), - readyService: ({ id }) => - orchestrator.get(id).pipe( - Effect.flatMap((entry) => entry.core.ready), - rpcError("readyService"), - ), + readyService: ({ id }) => orchestrator.ready(id).pipe(rpcError("readyService")), stopService: ({ id }) => orchestrator.stop(id).pipe(rpcError("stopService")), // A config-changing restart can adopt saved credentials, so it serializes with definition // changes that introduce or roll them back. @@ -739,7 +734,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { .get(id) .pipe( Effect.map((entry) => - entry.core.observation.pipe(Stream.mapEffect((value) => observe(entry, value))), + orchestrator.changes(id).pipe(Stream.mapEffect((value) => observe(entry, value))), ), ), ).pipe(Stream.mapError((cause) => stackError("followStatus", cause))), diff --git a/packages/stack/src/Rpc.ts b/packages/stack/src/Rpc.ts index 5696ff1f78..ec0738d96e 100644 --- a/packages/stack/src/Rpc.ts +++ b/packages/stack/src/Rpc.ts @@ -129,7 +129,6 @@ export const Observation = Schema.Struct({ Schema.Literals(["start", "stop", "restart", "storage", "destroy", "sleep"]), ), launchId: Schema.UndefinedOr(Schema.Int), - intentRevision: Schema.Int, wakeEnabled: Schema.Boolean, registered: Schema.Boolean, }); diff --git a/packages/stack/src/Service.integration.test.ts b/packages/stack/src/Service.integration.test.ts index 5947e64b4f..8f3990038e 100644 --- a/packages/stack/src/Service.integration.test.ts +++ b/packages/stack/src/Service.integration.test.ts @@ -1,18 +1,17 @@ import { describe, expect, it } from "@effect/vitest"; -import { Cause, Deferred, Effect, Exit, Fiber, Queue, Ref, Scope, Stream } from "effect"; +import { Deferred, Effect, Exit, Fiber, Option, Queue, Ref, Scope, Stream } from "effect"; +import { makeStandaloneService } from "../tests/standalone-service.ts"; +import type { Status } from "./Orchestrator.ts"; import { makeService, - ServiceDestroyed, ServiceError, ServiceLaunchError, - ServiceStaleLaunch, type RuntimeSession, type ServiceDefinition, - type ServiceInstance, - type ServiceObservation, } from "./Service.ts"; type Config = { readonly version: number }; +type Standalone = Effect.Success>>; interface ResourceState { readonly phase: "created" | "launched" | "stopping" | "stopped"; @@ -39,7 +38,6 @@ interface RuntimePlan { interface PreparationPlan { readonly gate: Deferred.Deferred; readonly started: Deferred.Deferred; - readonly completed: Deferred.Deferred; } const makeRuntimePlan = Effect.gen(function* () { @@ -68,76 +66,48 @@ const makePreparationPlan = Effect.gen(function* () { return { gate: yield* Deferred.make(), started: yield* Deferred.make(), - completed: yield* Deferred.make(), } satisfies PreparationPlan; }); const open = (deferred: Deferred.Deferred) => Deferred.succeed(deferred, undefined); -const waitForObservation = ( - service: ServiceInstance, - predicate: (observation: ServiceObservation) => boolean, -) => +const waitForStatus = (service: Standalone, predicate: (status: Status) => boolean) => service.observation.pipe( Stream.filter(predicate), Stream.runHead, Effect.asVoid, - Effect.forkScoped, + Effect.forkScoped({ startImmediately: true }), ); const makeDefinition = ( plans: Queue.Queue, - preparations?: Queue.Queue, - prepared?: Ref.Ref>, - invalid?: Ref.Ref, - launchFailure?: Ref.Ref, + options: { + readonly preparations?: Queue.Queue; + readonly invalid?: Ref.Ref; + } = {}, ): ServiceDefinition => ({ - prepare: - preparations === undefined && prepared === undefined && invalid === undefined - ? undefined - : (config) => { - const queue = preparations; - return Effect.gen(function* () { - if (invalid !== undefined && (yield* Ref.get(invalid))) { - return yield* new ServiceError({ - operation: "prepare", - message: "invalid configuration", - }); - } - if (prepared !== undefined) - yield* Ref.update(prepared, (values) => [...values, config]); - if (queue !== undefined) { - const preparation = yield* Queue.take(queue); - yield* Deferred.succeed(preparation.started, undefined); - yield* Deferred.await(preparation.gate); - yield* Deferred.succeed(preparation.completed, undefined); - } - }); - }, + prepare: () => + Effect.gen(function* () { + if (options.invalid !== undefined && (yield* Ref.get(options.invalid))) + return yield* new ServiceError({ operation: "prepare", message: "invalid configuration" }); + if (options.preparations === undefined) return; + const preparation = yield* Queue.take(options.preparations); + yield* open(preparation.started); + yield* Deferred.await(preparation.gate); + }), launch: (_context) => Effect.gen(function* () { const plan = yield* Queue.take(plans); - yield* Deferred.succeed(plan.launchStarted, undefined); + yield* open(plan.launchStarted); yield* Deferred.await(plan.launchGate); - if (launchFailure !== undefined && (yield* Ref.get(launchFailure))) { - yield* Ref.set(launchFailure, false); - return yield* new ServiceError({ operation: "launch", message: "launch failed" }); - } yield* Ref.update(plan.state, (state): ResourceState => ({ ...state, phase: "launched" })); const session: RuntimeSession = { health: Effect.gen(function* () { - const state = yield* Ref.get(plan.state); - if (state.health !== "not-started") { - return yield* new ServiceError({ - operation: "health", - message: "health started twice", - }); - } yield* Ref.update(plan.state, (value): ResourceState => ({ ...value, health: "starting", })); - yield* Deferred.succeed(plan.healthStarted, undefined); + yield* open(plan.healthStarted); yield* Deferred.await(plan.healthGate); yield* Ref.update(plan.state, (value): ResourceState => ({ ...value, @@ -150,24 +120,20 @@ const makeDefinition = ( ...value, phase: "stopping", })); - yield* Deferred.succeed(plan.stopStarted, undefined); - if (yield* Ref.get(plan.stopFailure)) { - yield* Ref.set(plan.stopFailure, false); + yield* open(plan.stopStarted); + if (yield* Ref.getAndSet(plan.stopFailure, false)) return yield* new ServiceError({ operation: "stop", message: "stop failed" }); - } - yield* Deferred.succeed(plan.stopRetryStarted, undefined); + yield* open(plan.stopRetryStarted); yield* Deferred.await(plan.stopGate); yield* Ref.update(plan.state, (value): ResourceState => ({ ...value, phase: "stopped" })); }), remove: Effect.gen(function* () { - yield* Deferred.succeed(plan.removeStarted, undefined); - if (yield* Ref.get(plan.removeFailure)) { - yield* Ref.set(plan.removeFailure, false); + yield* open(plan.removeStarted); + if (yield* Ref.getAndSet(plan.removeFailure, false)) return yield* new ServiceError({ operation: "remove", message: "exact cleanup failed", }); - } yield* Deferred.await(plan.removeGate); yield* Ref.update(plan.state, (value) => ({ ...value, removed: true })); }), @@ -177,61 +143,93 @@ const makeDefinition = ( removeData: () => Effect.void, }); -const makeFixture = Effect.gen(function* () { - const plans = yield* Queue.unbounded(); - const service = yield* makeService(makeDefinition(plans), { - id: "database-1", - config: { version: 17 }, +const makeFixture = (id: string, definition = makeDefinition) => + Effect.gen(function* () { + const plans = yield* Queue.unbounded(); + const service = yield* makeStandaloneService(definition(plans), { + id, + config: { version: 17 }, + }); + return { plans, service }; }); - return { plans, service }; -}); -const stopFixture = (service: ServiceInstance, plan: RuntimePlan) => +const partialFailure = new ServiceError({ operation: "launch", message: "initialization failed" }); + +/** Acquires a runtime, then fails the launch with it retained for cleanup. */ +const partialDefinition = (plans: Queue.Queue): ServiceDefinition => { + const base = makeDefinition(plans); + return { + launch: (context) => + base + .launch(context) + .pipe( + Effect.flatMap((runtime) => + Effect.fail(new ServiceLaunchError({ failure: partialFailure, runtime })), + ), + ), + removeData: base.removeData, + }; +}; + +/** Waits until the service's lifecycle records a pending destroy. */ +const awaitDestroyPending = (service: Standalone) => + service.observation.pipe( + Stream.filter((status) => status.destroyPending), + Stream.runHead, + ); + +const stopFixture = (service: Standalone, plan: RuntimePlan) => Effect.gen(function* () { yield* open(plan.stopGate); yield* open(plan.removeGate); yield* service.stop; }); -describe("service kernel", () => { - it.live("submits stop while launch is delayed and completes it after launch", () => - Effect.scoped( - Effect.gen(function* () { - const fixture = yield* makeFixture; - const plan = yield* makeRuntimePlan; - yield* Queue.offer(fixture.plans, plan); - const starting = yield* fixture.service.start.pipe(Effect.forkScoped); - yield* Deferred.await(plan.launchStarted); - - const stopping = yield* fixture.service.stop.pipe(Effect.forkScoped); - expect((yield* fixture.service.get).lifecycle).toBe("starting"); - yield* open(plan.launchGate); - yield* Fiber.join(starting); - yield* Deferred.await(plan.stopStarted); - yield* open(plan.stopGate); - yield* Deferred.await(plan.removeStarted); - yield* open(plan.removeGate); - yield* Fiber.join(stopping); +describe("service execution", () => { + it.live( + "stops a launch requested to stop mid-launch once its runtime exists, then cleans it", + () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture("database-mid-launch"); + const plan = yield* makeRuntimePlan; + yield* Queue.offer(fixture.plans, plan); + const starting = yield* fixture.service.start.pipe(Effect.exit, Effect.forkScoped); + yield* Deferred.await(plan.launchStarted); + + const stopRequested = yield* waitForStatus( + fixture.service, + (status) => status.lifecycle === "stopping", + ); + const stopping = yield* fixture.service.stop.pipe(Effect.forkScoped); + yield* Fiber.join(stopRequested); + yield* open(plan.launchGate); + yield* Deferred.await(plan.stopStarted); + yield* open(plan.stopGate); + yield* Deferred.await(plan.removeStarted); + yield* open(plan.removeGate); + yield* Fiber.join(stopping); - expect((yield* fixture.service.get).lifecycle).toBe("stopped"); - expect((yield* Ref.get(plan.state)).removed).toBe(true); - }), - ), + expect(Exit.isFailure(yield* Fiber.join(starting))).toBe(true); + expect((yield* fixture.service.get).lifecycle).toBe("stopped"); + expect((yield* Ref.get(plan.state)).removed).toBe(true); + }), + ), ); - it.live("shares one health program and a canceled readiness waiter does not poison it", () => + it.live("shares one health check and a cancelled readiness caller does not poison it", () => Effect.scoped( Effect.gen(function* () { - const fixture = yield* makeFixture; + const fixture = yield* makeFixture("database-shared-health"); const plan = yield* makeRuntimePlan; yield* Queue.offer(fixture.plans, plan); yield* open(plan.launchGate); yield* fixture.service.start; yield* Deferred.await(plan.healthStarted); - const canceled = yield* fixture.service.ready.pipe(Effect.forkScoped); + const cancelled = yield* fixture.service.ready.pipe(Effect.forkScoped); const ready = yield* fixture.service.ready.pipe(Effect.forkScoped); - yield* Fiber.interrupt(canceled); + yield* Fiber.interrupt(cancelled); yield* open(plan.healthGate); expect(Exit.isSuccess(yield* Fiber.await(ready))).toBe(true); expect((yield* Ref.get(plan.state)).health).toBe("healthy"); @@ -241,116 +239,75 @@ describe("service kernel", () => { ), ); - it.live("cancels after preparation while waiting for the lifecycle gate", () => - Effect.scoped( - Effect.gen(function* () { - const plans = yield* Queue.unbounded(); - const preparations = yield* Queue.unbounded(); - const preparation = yield* makePreparationPlan; - const retryPreparation = yield* makePreparationPlan; - yield* Queue.offer(preparations, preparation); - yield* Queue.offer(preparations, retryPreparation); - const service = yield* makeService(makeDefinition(plans, preparations), { - id: "database-gate-wait", - config: { version: 17 }, - }); - const plan = yield* makeRuntimePlan; - yield* Queue.offer(plans, plan); - - const storageStarted = yield* Deferred.make(); - const storageGate = yield* Deferred.make(); - const storage = yield* service - .storage( - Effect.gen(function* () { - yield* Deferred.succeed(storageStarted, undefined); - yield* Deferred.await(storageGate); - }), - ) - .pipe(Effect.forkScoped); - yield* Deferred.await(storageStarted); - - const start = yield* service.start.pipe(Effect.forkScoped); - yield* Deferred.await(preparation.started); - yield* open(preparation.gate); - yield* Deferred.await(preparation.completed); - yield* Fiber.interrupt(start); - yield* open(storageGate); - yield* Fiber.join(storage); - - expect(yield* Deferred.isDone(plan.launchStarted)).toBe(false); - const retry = yield* service.start.pipe(Effect.forkScoped); - yield* Deferred.await(retryPreparation.started); - yield* open(retryPreparation.gate); - yield* open(plan.launchGate); - yield* Fiber.join(retry); - expect((yield* service.get).lifecycle).toBe("running"); - yield* stopFixture(service, plan); - }), - ), - ); - - it.live("records a failed wake preparation on the observation", () => + it.live("records a failed preparation on the observation and stays wakeable", () => Effect.scoped( Effect.gen(function* () { const plans = yield* Queue.unbounded(); const invalid = yield* Ref.make(true); - const service = yield* makeService(makeDefinition(plans, undefined, undefined, invalid), { - id: "database-wake-prepare", + const service = yield* makeStandaloneService(makeDefinition(plans, { invalid }), { + id: "database-prepare-failure", config: { version: 17 }, }); - yield* service.arm; - const failure = yield* service.startAt(0, undefined, true).pipe(Effect.flip); - expect(failure).toBeInstanceOf(ServiceError); - const observation = yield* service.get; - expect(observation.lifecycle).toBe("stopped"); - expect(observation.wakeEnabled).toBe(true); - expect(observation.error?.message).toBe("invalid configuration"); + const failure = yield* service.start.pipe(Effect.flip); + expect(failure.message).toContain("invalid configuration"); + expect(yield* service.get).toMatchObject({ + lifecycle: "stopped", + wakeEnabled: true, + error: { message: "invalid configuration" }, + }); }), ), ); - it.live("keeps a stale preparation failure off a relaunched observation", () => + it.live("keeps a stopped launch's late preparation failure off its successor", () => Effect.scoped( Effect.gen(function* () { const plans = yield* Queue.unbounded(); const staleStarted = yield* Deferred.make(); const staleGate = yield* Deferred.make(); const attempts = yield* Ref.make(0); - const service = yield* makeService( + const service = yield* makeStandaloneService( { ...makeDefinition(plans), + // Uninterruptible, so the stop has to wait it out and its failure arrives late. prepare: () => Effect.gen(function* () { if ((yield* Ref.updateAndGet(attempts, (value) => value + 1)) > 1) return; - yield* Deferred.succeed(staleStarted, undefined); + yield* open(staleStarted); yield* Deferred.await(staleGate); return yield* new ServiceError({ operation: "prepare", message: "stale preparation", }); - }), + }).pipe(Effect.uninterruptible), }, { id: "database-stale-failure", config: { version: 17 } }, ); const plan = yield* makeRuntimePlan; yield* Queue.offer(plans, plan); yield* open(plan.launchGate); - const stale = yield* service.start.pipe(Effect.forkScoped); + const stale = yield* service.start.pipe(Effect.exit, Effect.forkScoped); yield* Deferred.await(staleStarted); - yield* service.start; - expect((yield* service.get).lifecycle).toBe("running"); - + // The stop of generation 1 is admitted, and waiting on its preparation, before it fails. + const stopAdmitted = yield* waitForStatus( + service, + (status) => status.lifecycle === "stopping", + ); + const restarted = yield* service.restart().pipe(Effect.forkScoped); + yield* Fiber.join(stopAdmitted); yield* open(staleGate); - expect(Exit.isFailure(yield* Fiber.await(stale))).toBe(true); - const observation = yield* service.get; - expect(observation.lifecycle).toBe("running"); - expect(observation.error).toBeUndefined(); + yield* Fiber.join(restarted); + expect((yield* service.get).launchId).toBe(2); + + // The earlier start is satisfied by the restarted generation. + expect(Exit.isSuccess(yield* Fiber.join(stale))).toBe(true); + expect(yield* service.get).toMatchObject({ lifecycle: "running", error: undefined }); yield* stopFixture(service, plan); }), ), ); - it.live("does not prepare concurrent starts more than once", () => + it.live("prepares and launches once for concurrent starts", () => Effect.scoped( Effect.gen(function* () { const plans = yield* Queue.unbounded(); @@ -359,8 +316,8 @@ describe("service kernel", () => { const duplicatePreparation = yield* makePreparationPlan; yield* Queue.offer(preparations, preparation); yield* Queue.offer(preparations, duplicatePreparation); - const service = yield* makeService(makeDefinition(plans, preparations), { - id: "database-concurrent-prepare", + const service = yield* makeStandaloneService(makeDefinition(plans, { preparations }), { + id: "database-concurrent-start", config: { version: 17 }, }); const plan = yield* makeRuntimePlan; @@ -368,150 +325,97 @@ describe("service kernel", () => { const first = yield* service.start.pipe(Effect.forkScoped); yield* Deferred.await(preparation.started); - yield* open(preparation.gate); - yield* Deferred.await(preparation.completed); - yield* Deferred.await(plan.launchStarted); const second = yield* service.start.pipe(Effect.forkScoped({ startImmediately: true })); + yield* open(preparation.gate); yield* open(plan.launchGate); yield* Fiber.join(first); - expect(yield* Deferred.isDone(duplicatePreparation.started)).toBe(false); yield* Fiber.join(second); + expect(yield* Deferred.isDone(duplicatePreparation.started)).toBe(false); expect((yield* service.get).lifecycle).toBe("running"); yield* stopFixture(service, plan); }), ), ); - it.live("reprepares a queued candidate after the current launch fails", () => - Effect.scoped( - Effect.gen(function* () { - const plans = yield* Queue.unbounded(); - const preparations = yield* Queue.unbounded(); - const firstPreparation = yield* makePreparationPlan; - const secondPreparation = yield* makePreparationPlan; - yield* Queue.offer(preparations, firstPreparation); - yield* Queue.offer(preparations, secondPreparation); - const launchFailure = yield* Ref.make(true); - const service = yield* makeService( - makeDefinition(plans, preparations, undefined, undefined, launchFailure), - { id: "database-retry-prepare", config: { version: 17 } }, - ); - const firstPlan = yield* makeRuntimePlan; - const secondPlan = yield* makeRuntimePlan; - yield* Queue.offer(plans, firstPlan); - yield* Queue.offer(plans, secondPlan); - - const first = yield* service.start.pipe(Effect.forkScoped); - yield* Deferred.await(firstPreparation.started); - yield* open(firstPreparation.gate); - yield* Deferred.await(firstPlan.launchStarted); - const second = yield* service - .startAt(0, { version: 18 }, false, Effect.void) - .pipe(Effect.forkScoped({ startImmediately: true })); - yield* open(firstPlan.launchGate); - expect(Exit.isFailure(yield* Fiber.await(first))).toBe(true); - yield* Deferred.await(secondPreparation.started); - yield* open(secondPreparation.gate); - yield* open(secondPlan.launchGate); - yield* Fiber.join(second); - expect((yield* service.get).config.version).toBe(18); - yield* stopFixture(service, secondPlan); - }), - ), - ); - - it.live("invalidates a prepared start when stop is admitted before preparation finishes", () => + it.live("skips the launch when a stop arrives during preparation", () => Effect.scoped( Effect.gen(function* () { const plans = yield* Queue.unbounded(); const preparations = yield* Queue.unbounded(); const preparation = yield* makePreparationPlan; yield* Queue.offer(preparations, preparation); - const service = yield* makeService(makeDefinition(plans, preparations), { - id: "database-stale-preparation", + const service = yield* makeStandaloneService(makeDefinition(plans, { preparations }), { + id: "database-stop-in-preparation", config: { version: 17 }, }); const plan = yield* makeRuntimePlan; yield* Queue.offer(plans, plan); - const start = yield* service.start.pipe(Effect.forkScoped); + const start = yield* service.start.pipe(Effect.exit, Effect.forkScoped); yield* Deferred.await(preparation.started); yield* service.stop; yield* open(preparation.gate); - const result = yield* Fiber.await(start); - expect(Exit.isFailure(result)).toBe(true); + expect(Exit.isFailure(yield* Fiber.join(start))).toBe(true); expect(yield* Deferred.isDone(plan.launchStarted)).toBe(false); expect((yield* service.get).lifecycle).toBe("stopped"); }), ), ); - it.live( - "stops and cleans up the running workload when a restart is destroyed mid-preparation and completes late", - () => - Effect.scoped( - Effect.gen(function* () { - const plans = yield* Queue.unbounded(); - const preparations = yield* Queue.unbounded(); - const initialPreparation = yield* makePreparationPlan; - const restartPreparation = yield* makePreparationPlan; - yield* open(initialPreparation.gate); - yield* Queue.offer(preparations, initialPreparation); - yield* Queue.offer(preparations, restartPreparation); - const service = yield* makeService(makeDefinition(plans, preparations), { - id: "database-late-restart", - config: { version: 17 }, - }); - const original = yield* makeRuntimePlan; - const superseded = yield* makeRuntimePlan; - yield* Queue.offer(plans, original); - yield* Queue.offer(plans, superseded); - yield* open(original.launchGate); - yield* service.start; - expect((yield* service.get).lifecycle).toBe("running"); + it.live("never launches a restart whose service is destroyed while it prepares", () => + Effect.scoped( + Effect.gen(function* () { + const plans = yield* Queue.unbounded(); + const preparations = yield* Queue.unbounded(); + const initialPreparation = yield* makePreparationPlan; + const restartPreparation = yield* makePreparationPlan; + yield* open(initialPreparation.gate); + yield* Queue.offer(preparations, initialPreparation); + yield* Queue.offer(preparations, restartPreparation); + const service = yield* makeStandaloneService(makeDefinition(plans, { preparations }), { + id: "database-late-restart", + config: { version: 17 }, + }); + const original = yield* makeRuntimePlan; + const superseded = yield* makeRuntimePlan; + yield* Queue.offer(plans, original); + yield* Queue.offer(plans, superseded); + yield* open(original.launchGate); + yield* service.start; - const restarting = yield* service.restart({ version: 18 }).pipe(Effect.forkScoped); - yield* Deferred.await(restartPreparation.started); + const restarting = yield* service.restart({ version: 18 }).pipe(Effect.forkScoped); + yield* Deferred.await(restartPreparation.started); + yield* open(original.stopGate); + yield* open(original.removeGate); + yield* service.destroy; + expect((yield* service.execution).registered).toBe(false); + expect((yield* Ref.get(original.state)).removed).toBe(true); - yield* open(original.stopGate); - yield* open(original.removeGate); - yield* service.destroy; - expect((yield* service.get).registered).toBe(false); - expect((yield* Ref.get(original.state)).removed).toBe(true); - - // The superseded restart's own preparation only resolves after the destroy above, so its - // launch attempt (if it ran at all) would be the one completing late. - yield* open(restartPreparation.gate); - const result = yield* Fiber.await(restarting); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) - expect(result.cause.reasons.find(Cause.isFailReason)?.error).toBeInstanceOf( - ServiceDestroyed, - ); - expect(yield* Deferred.isDone(superseded.launchStarted)).toBe(false); - }), - ), + yield* open(restartPreparation.gate); + expect(Exit.isFailure(yield* Fiber.await(restarting))).toBe(true); + expect(yield* Deferred.isDone(superseded.launchStarted)).toBe(false); + }), + ), ); - it.live("records a launch failure as stopped and can launch successfully on retry", () => + it.live("records a launch failure as stopped and launches successfully on retry", () => Effect.scoped( Effect.gen(function* () { const plans = yield* Queue.unbounded(); const failLaunch = yield* Ref.make(true); const definition = makeDefinition(plans); - const service = yield* makeService( + const service = yield* makeStandaloneService( { launch: (context) => - Effect.gen(function* () { - if (yield* Ref.get(failLaunch)) { - yield* Ref.set(failLaunch, false); - return yield* new ServiceError({ - operation: "launch", - message: "binary unavailable", - }); - } - return yield* definition.launch(context); - }), + Ref.getAndSet(failLaunch, false).pipe( + Effect.flatMap((fail) => + fail + ? Effect.fail( + new ServiceError({ operation: "launch", message: "binary unavailable" }), + ) + : definition.launch(context), + ), + ), removeData: definition.removeData, }, { id: "database-launch-retry", config: { version: 17 } }, @@ -519,10 +423,11 @@ describe("service kernel", () => { const plan = yield* makeRuntimePlan; yield* Queue.offer(plans, plan); - const failed = yield* service.start.pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* service.get).lifecycle).toBe("stopped"); - expect((yield* service.get).error?.operation).toBe("launch"); + expect(Exit.isFailure(yield* service.start.pipe(Effect.exit))).toBe(true); + expect(yield* service.get).toMatchObject({ + lifecycle: "stopped", + error: { operation: "launch" }, + }); expect((yield* Ref.get(plan.state)).phase).toBe("created"); yield* open(plan.launchGate); @@ -533,204 +438,146 @@ describe("service kernel", () => { ), ); - it.live("retains a runtime acquired before launch failure for exact cleanup", () => + it.live("retains a runtime acquired before a launch failure for exact cleanup", () => Effect.scoped( Effect.gen(function* () { - const fixture = yield* makeFixture; + const failure = partialFailure; + const partial = yield* makeFixture("database-partial", partialDefinition); const plan = yield* makeRuntimePlan; - const base = makeDefinition(fixture.plans); - const failure = new ServiceError({ operation: "launch", message: "initialization failed" }); - const partial = yield* makeService( - { - launch: (context) => - base - .launch(context) - .pipe( - Effect.flatMap((runtime) => - Effect.fail(new ServiceLaunchError({ failure, runtime })), - ), - ), - removeData: base.removeData, - }, - { id: "database-partial", config: { version: 17 } }, - ); - yield* Queue.offer(fixture.plans, plan); + yield* Queue.offer(partial.plans, plan); yield* open(plan.launchGate); yield* Ref.set(plan.stopFailure, true); - const failed = yield* partial.start.pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* partial.get).lifecycle).toBe("stopping"); - expect((yield* partial.get).error).toEqual(failure); - expect((yield* partial.get).cleanupError).toBeDefined(); + expect(Exit.isFailure(yield* partial.service.start.pipe(Effect.exit))).toBe(true); + expect(yield* partial.service.get).toMatchObject({ lifecycle: "stopping", error: failure }); + expect((yield* partial.service.get).cleanupError).toBeDefined(); expect((yield* Ref.get(plan.state)).removed).toBe(false); - const retry = yield* partial.stop.pipe(Effect.forkScoped); + const retry = yield* partial.service.stop.pipe(Effect.forkScoped); yield* Deferred.await(plan.stopRetryStarted); yield* open(plan.stopGate); yield* Deferred.await(plan.removeStarted); yield* open(plan.removeGate); yield* Fiber.join(retry); - expect((yield* partial.get).lifecycle).toBe("stopped"); + expect(yield* partial.service.get).toMatchObject({ + lifecycle: "stopped", + cleanupError: undefined, + }); expect((yield* Ref.get(plan.state)).removed).toBe(true); - const plans2 = yield* Queue.unbounded(); - const plan2 = yield* makeRuntimePlan; - const base2 = makeDefinition(plans2); - const cleanFailure = new ServiceError({ - operation: "launch", - message: "clean partial failure", - }); - const cleaned = yield* makeService( - { - launch: (context) => - base2 - .launch(context) - .pipe( - Effect.flatMap((runtime) => - Effect.fail(new ServiceLaunchError({ failure: cleanFailure, runtime })), - ), - ), - removeData: base2.removeData, - }, - { id: "database-partial-clean", config: { version: 17 } }, - ); - yield* Queue.offer(plans2, plan2); - yield* open(plan2.launchGate); - yield* open(plan2.stopGate); - yield* open(plan2.removeGate); - expect(Exit.isFailure(yield* cleaned.start.pipe(Effect.exit))).toBe(true); - expect((yield* cleaned.get).lifecycle).toBe("stopped"); - expect((yield* Ref.get(plan2.state)).removed).toBe(true); + const cleaned = yield* makeFixture("database-partial-clean", partialDefinition); + const cleanPlan = yield* makeRuntimePlan; + yield* Queue.offer(cleaned.plans, cleanPlan); + yield* open(cleanPlan.launchGate); + yield* open(cleanPlan.stopGate); + yield* open(cleanPlan.removeGate); + expect(Exit.isFailure(yield* cleaned.service.start.pipe(Effect.exit))).toBe(true); + expect((yield* cleaned.service.get).lifecycle).toBe("stopped"); + expect((yield* Ref.get(cleanPlan.state)).removed).toBe(true); }), ), ); - it.live("admits a prepared start after restart and does not relaunch", () => - Effect.scoped( - Effect.gen(function* () { - const plans = yield* Queue.unbounded(); - const preparations = yield* Queue.unbounded(); - const queuedStartPreparation = yield* makePreparationPlan; - const restartPreparation = yield* makePreparationPlan; - yield* Queue.offer(preparations, queuedStartPreparation); - yield* Queue.offer(preparations, restartPreparation); - const service = yield* makeService(makeDefinition(plans, preparations), { - id: "database-start-after-restart", - config: { version: 17 }, - }); - const first = yield* makeRuntimePlan; - const unused = yield* makeRuntimePlan; - yield* Queue.offer(plans, first); - yield* Queue.offer(plans, unused); + it.live( + "keeps a retained runtime pending cleanup, refusing storage, until a stop retry confirms its removal", + () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture("database-retained", partialDefinition); + const plan = yield* makeRuntimePlan; + yield* Queue.offer(fixture.plans, plan); + yield* open(plan.launchGate); + yield* Ref.set(plan.stopFailure, true); + expect(Exit.isFailure(yield* fixture.service.start.pipe(Effect.exit))).toBe(true); + + yield* Ref.set(plan.stopFailure, true); + expect(Exit.isFailure(yield* fixture.service.stop.pipe(Effect.exit))).toBe(true); + expect((yield* fixture.service.get).lifecycle).toBe("stopping"); + expect( + Exit.isFailure(yield* fixture.service.storage(Effect.void).pipe(Effect.exit)), + ).toBe(true); + + yield* open(plan.stopGate); + yield* open(plan.removeGate); + yield* fixture.service.stop; + expect((yield* fixture.service.get).lifecycle).toBe("stopped"); + expect((yield* Ref.get(plan.state)).removed).toBe(true); + }), + ), + ); - const queuedStart = yield* service.start.pipe(Effect.forkScoped); - yield* Deferred.await(queuedStartPreparation.started); + it.live( + "fails a restart whose stop fails instead of waiting, and retries the cleanup on stop", + () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture("database-restart-stop-failure"); + const plan = yield* makeRuntimePlan; + yield* Queue.offer(fixture.plans, plan); + yield* open(plan.launchGate); + yield* fixture.service.start; + yield* Ref.set(plan.stopFailure, true); - const restart = yield* service.restart().pipe(Effect.forkScoped); - yield* Deferred.await(restartPreparation.started); - yield* open(restartPreparation.gate); - yield* open(first.launchGate); - yield* Fiber.join(restart); - expect((yield* service.get).lifecycle).toBe("running"); + const failed = yield* fixture.service + .restart() + .pipe(Effect.flip, Effect.timeoutOption("5 seconds")); + expect(Option.getOrUndefined(failed)?.message).toContain("stop failed"); + expect((yield* fixture.service.get).lifecycle).toBe("stopping"); - yield* open(queuedStartPreparation.gate); - expect(Exit.isSuccess(yield* Fiber.await(queuedStart))).toBe(true); - expect((yield* service.get).launchId).toBe(1); - expect(yield* Deferred.isDone(unused.launchStarted)).toBe(false); - yield* stopFixture(service, first); - }), - ), + yield* stopFixture(fixture.service, plan); + expect((yield* fixture.service.get).lifecycle).toBe("stopped"); + expect((yield* Ref.get(plan.state)).removed).toBe(true); + }), + ), ); - it.live("accepts a prepared start queued during restart once restart is running", () => + it.live("never lets a late stop of an older generation touch a newer session", () => Effect.scoped( Effect.gen(function* () { const plans = yield* Queue.unbounded(); - const preparations = yield* Queue.unbounded(); - const initialPreparation = yield* makePreparationPlan; - const queuedStartPreparation = yield* makePreparationPlan; - const restartPreparation = yield* makePreparationPlan; - yield* open(initialPreparation.gate); - yield* Queue.offer(preparations, initialPreparation); - yield* Queue.offer(preparations, restartPreparation); - yield* Queue.offer(preparations, queuedStartPreparation); - const service = yield* makeService(makeDefinition(plans, preparations), { - id: "database-restart-queue", + const service = yield* makeService(makeDefinition(plans), { + id: "database-late-stop", config: { version: 17 }, + report: () => Effect.void, }); const first = yield* makeRuntimePlan; const second = yield* makeRuntimePlan; yield* Queue.offer(plans, first); yield* Queue.offer(plans, second); - yield* open(first.launchGate); - yield* service.start; - - const restartRunning = yield* waitForObservation( - service, - (value) => value.currentOperation === "restart", - ); - const restart = yield* service.restart().pipe(Effect.forkScoped); - yield* Deferred.await(restartPreparation.started); - yield* open(restartPreparation.gate); - yield* Fiber.join(restartRunning); - const queuedStart = yield* service.start.pipe(Effect.forkScoped); - yield* Deferred.await(queuedStartPreparation.started); - yield* Deferred.await(first.stopStarted); - yield* open(first.stopGate); - yield* Deferred.await(first.removeStarted); - yield* open(first.removeGate); - yield* Deferred.await(second.launchStarted); - yield* open(queuedStartPreparation.gate); - yield* open(second.launchGate); - yield* Fiber.join(restart); - expect(Exit.isSuccess(yield* Fiber.await(queuedStart))).toBe(true); - expect((yield* service.get).lifecycle).toBe("running"); - yield* stopFixture(service, second); - }), - ), - ); - - it.live("keeps an admitted launch alive after its caller is canceled", () => - Effect.scoped( - Effect.gen(function* () { - const fixture = yield* makeFixture; - const plan = yield* makeRuntimePlan; - yield* Queue.offer(fixture.plans, plan); - const running = yield* waitForObservation( - fixture.service, - (value) => value.lifecycle === "running", - ); - const caller = yield* fixture.service.start.pipe(Effect.forkScoped); - yield* Deferred.await(plan.launchStarted); - yield* Fiber.interrupt(caller); - yield* open(plan.launchGate); - yield* Fiber.join(running); - expect((yield* Ref.get(plan.state)).phase).toBe("launched"); - yield* stopFixture(fixture.service, plan); + for (const plan of [first, second]) + for (const gate of [plan.launchGate, plan.healthGate, plan.stopGate, plan.removeGate]) + yield* open(gate); + const stopOptions = { operation: "stop", discard: false } as const; + + yield* service.launch(1, { version: 17 }); + yield* service.stop(1, stopOptions); + yield* service.launch(2, { version: 17 }); + yield* service.stop(1, stopOptions); + + expect((yield* Ref.get(second.state)).phase).toBe("launched"); + yield* service.stop(2, stopOptions); + expect((yield* Ref.get(second.state)).removed).toBe(true); }), ), ); it.live( - "keeps the handle's scope and its resources alive through a cold launch after the caller is canceled", + "keeps a launch, and the resources its scope owns, alive after its caller is cancelled", () => Effect.scoped( Effect.gen(function* () { const resourceClosed = yield* Deferred.make(); const launchStarted = yield* Deferred.make(); const launchGate = yield* Deferred.make(); - const service = yield* makeService( + const service = yield* makeStandaloneService( { launch: (context) => Effect.gen(function* () { - // A real resource owned by the launch's execution handle, not just the fake - // plan's own bookkeeping: closing the handle early would settle this finalizer. yield* Scope.addFinalizer( context.scope, - Deferred.succeed(resourceClosed, undefined).pipe(Effect.asVoid), + open(resourceClosed).pipe(Effect.asVoid), ); - yield* Deferred.succeed(launchStarted, undefined); + yield* open(launchStarted); yield* Deferred.await(launchGate); return { health: Effect.void, @@ -746,9 +593,8 @@ describe("service kernel", () => { const caller = yield* service.start.pipe(Effect.forkScoped); yield* Deferred.await(launchStarted); yield* Fiber.interrupt(caller); - // The caller gave up mid-launch; the owner fiber and the scope it owns must stay live. expect(yield* Deferred.isDone(resourceClosed)).toBe(false); - yield* Deferred.succeed(launchGate, undefined); + yield* open(launchGate); yield* service.ready; expect((yield* service.get).lifecycle).toBe("running"); expect(yield* Deferred.isDone(resourceClosed)).toBe(false); @@ -758,213 +604,146 @@ describe("service kernel", () => { ), ); - it.live("lets independent instances progress concurrently", () => + it.live("keeps exact cleanup authority after a failed stop and retries only what remains", () => Effect.scoped( Effect.gen(function* () { - const leftFixture = yield* makeFixture; - const rightFixture = yield* makeFixture; - const leftPlan = yield* makeRuntimePlan; - const rightPlan = yield* makeRuntimePlan; - yield* Queue.offer(leftFixture.plans, leftPlan); - yield* Queue.offer(rightFixture.plans, rightPlan); - const leftRunning = yield* waitForObservation( - leftFixture.service, - (value) => value.lifecycle === "running", - ); - const rightRunning = yield* waitForObservation( - rightFixture.service, - (value) => value.lifecycle === "running", - ); - const leftStart = yield* leftFixture.service.start.pipe(Effect.forkScoped); - const rightStart = yield* rightFixture.service.start.pipe(Effect.forkScoped); - yield* Deferred.await(leftPlan.launchStarted); - yield* Deferred.await(rightPlan.launchStarted); - yield* open(leftPlan.launchGate); - yield* Fiber.join(leftRunning); - expect((yield* leftFixture.service.get).lifecycle).toBe("running"); - expect((yield* rightFixture.service.get).lifecycle).toBe("starting"); - yield* open(rightPlan.launchGate); - yield* Fiber.join(leftStart); - yield* Fiber.join(rightRunning); - yield* Fiber.join(rightStart); - yield* stopFixture(leftFixture.service, leftPlan); - yield* stopFixture(rightFixture.service, rightPlan); - }), - ), - ); - - it.live("keeps exact cleanup authority after failure and retries only remaining cleanup", () => - Effect.scoped( - Effect.gen(function* () { - const fixture = yield* makeFixture; + const fixture = yield* makeFixture("database-cleanup-retry"); const plan = yield* makeRuntimePlan; yield* Queue.offer(fixture.plans, plan); yield* open(plan.launchGate); yield* fixture.service.start; yield* open(plan.stopGate); yield* Ref.set(plan.removeFailure, true); - const failed = yield* fixture.service.stop.pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* fixture.service.get).lifecycle).toBe("stopping"); + expect(Exit.isFailure(yield* fixture.service.stop.pipe(Effect.exit))).toBe(true); + expect(yield* fixture.service.get).toMatchObject({ lifecycle: "stopping" }); + expect((yield* fixture.service.get).cleanupError).toBeDefined(); expect((yield* Ref.get(plan.state)).removed).toBe(false); + + const retry = yield* fixture.service.stop.pipe(Effect.forkScoped); + yield* Deferred.await(plan.removeStarted); + expect((yield* fixture.service.get).cleanupError).toBeDefined(); yield* open(plan.removeGate); - yield* fixture.service.stop; - expect((yield* fixture.service.get).lifecycle).toBe("stopped"); + yield* Fiber.join(retry); + expect(yield* fixture.service.get).toMatchObject({ + lifecycle: "stopped", + cleanupError: undefined, + }); expect((yield* Ref.get(plan.state)).removed).toBe(true); }), ), ); - it.live("keeps registration after data removal failure", () => - Effect.scoped( - Effect.gen(function* () { - const fixture = yield* makeFixture; - const plan = yield* makeRuntimePlan; - const dataFailure = yield* Ref.make(true); - const service = yield* makeService( - { - launch: makeDefinition(fixture.plans).launch, - removeData: () => - Effect.gen(function* () { - if (yield* Ref.get(dataFailure)) { - yield* Ref.set(dataFailure, false); - return yield* new ServiceError({ operation: "data", message: "retry" }); - } - }), - }, - { id: "database-data", config: { version: 17 } }, - ); - yield* Queue.offer(fixture.plans, plan); - yield* open(plan.launchGate); - yield* service.start; - yield* stopFixture(service, plan); - expect(Exit.isFailure(yield* service.destroy.pipe(Effect.exit))).toBe(true); - expect((yield* service.get).registered).toBe(true); - yield* service.destroy; - expect((yield* service.get).registered).toBe(false); - }), - ), + it.live( + "keeps the registration and data of a running instance whose destroy fails to stop it", + () => + Effect.scoped( + Effect.gen(function* () { + const plans = yield* Queue.unbounded(); + const dataRemoved = yield* Ref.make(false); + const service = yield* makeStandaloneService( + { launch: makeDefinition(plans).launch, removeData: () => Ref.set(dataRemoved, true) }, + { id: "database-destroy-stop-failure", config: { version: 17 } }, + ); + const plan = yield* makeRuntimePlan; + yield* Queue.offer(plans, plan); + yield* open(plan.launchGate); + yield* service.start; + yield* Ref.set(plan.stopFailure, true); + + expect(Exit.isFailure(yield* service.destroy.pipe(Effect.exit))).toBe(true); + expect(yield* service.get).toMatchObject({ lifecycle: "stopping", registered: true }); + expect(yield* Ref.get(dataRemoved)).toBe(false); + + yield* open(plan.stopGate); + yield* open(plan.removeGate); + yield* service.destroy; + expect(yield* Ref.get(dataRemoved)).toBe(true); + expect((yield* service.execution).registered).toBe(false); + }), + ), ); - it.live("destroys a running instance and retains registration after stop failure", () => + it.live("destroys an instance once the storage operation that owns it finishes", () => Effect.scoped( Effect.gen(function* () { - const fixture = yield* makeFixture; - const plan = yield* makeRuntimePlan; - yield* Queue.offer(fixture.plans, plan); - yield* open(plan.launchGate); - yield* fixture.service.start; - yield* open(plan.stopGate); - yield* open(plan.removeGate); - yield* fixture.service.destroy; - expect((yield* fixture.service.get).registered).toBe(false); - expect((yield* Ref.get(plan.state)).removed).toBe(true); - const plans = yield* Queue.unbounded(); - const dataRemoved = yield* Ref.make(false); - const service = yield* makeService( + const storageStarted = yield* Deferred.make(); + const storageGate = yield* Deferred.make(); + const removalStarted = yield* Deferred.make(); + const removalGate = yield* Deferred.make(); + const service = yield* makeStandaloneService( { launch: makeDefinition(plans).launch, - removeData: () => Ref.set(dataRemoved, true), + removeData: () => + open(removalStarted).pipe(Effect.andThen(Deferred.await(removalGate))), }, - { id: "database-destroy-stop-failure", config: { version: 17 } }, + { id: "database-storage", config: { version: 17 } }, ); - const failedPlan = yield* makeRuntimePlan; - yield* Queue.offer(plans, failedPlan); - yield* open(failedPlan.launchGate); - yield* service.start; - yield* Ref.set(failedPlan.stopFailure, true); - - expect(Exit.isFailure(yield* service.destroy.pipe(Effect.exit))).toBe(true); - expect((yield* service.get).lifecycle).toBe("stopping"); - expect((yield* service.get).registered).toBe(true); - expect(yield* Ref.get(dataRemoved)).toBe(false); - - yield* open(failedPlan.stopGate); - yield* open(failedPlan.removeGate); - yield* service.stop; - yield* service.destroy; - expect(yield* Ref.get(dataRemoved)).toBe(true); - expect((yield* service.get).registered).toBe(false); - }), - ), - ); + const storage = yield* service + .storage(open(storageStarted).pipe(Effect.andThen(Deferred.await(storageGate)))) + .pipe(Effect.forkScoped); + yield* Deferred.await(storageStarted); + expect((yield* service.get).currentOperation).toBe("storage"); - it.live("reports a destroyed instance as ServiceDestroyed to readiness callers", () => - Effect.scoped( - Effect.gen(function* () { - const fixture = yield* makeFixture; - yield* fixture.service.destroy; - const result = yield* fixture.service.ready.pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const failure = result.cause.reasons.find(Cause.isFailReason); - expect(failure !== undefined && failure.error instanceof ServiceDestroyed).toBe(true); - } - }), - ), - ); + const destroyed = yield* service.destroy.pipe(Effect.forkScoped); + yield* awaitDestroyPending(service); + expect((yield* service.execution).registered).toBe(true); + yield* open(storageGate); + yield* Fiber.join(storage); + yield* Deferred.await(removalStarted); + const refused = yield* service.storage(Effect.void).pipe(Effect.flip); + expect(refused.message).toBe("database-storage is being destroyed"); - it.live("waits for a pending launch before checking readiness", () => - Effect.scoped( - Effect.gen(function* () { - const fixture = yield* makeFixture; - const plan = yield* makeRuntimePlan; - yield* Queue.offer(fixture.plans, plan); - const start = yield* fixture.service.start.pipe(Effect.forkScoped); - yield* Deferred.await(plan.launchStarted); - const ready = yield* fixture.service.ready.pipe( - Effect.forkScoped({ startImmediately: true }), - ); - yield* open(plan.launchGate); - yield* Deferred.await(plan.healthStarted); - yield* open(plan.healthGate); - expect(Exit.isSuccess(yield* Fiber.await(ready))).toBe(true); - yield* Fiber.join(start); - yield* stopFixture(fixture.service, plan); + yield* open(removalGate); + yield* Fiber.join(destroyed); + expect((yield* service.execution).registered).toBe(false); }), ), ); - it.live("serializes storage and destroy through the same gate", () => + it.live("keeps an instance usable when a destroy waiting behind storage is interrupted", () => Effect.scoped( Effect.gen(function* () { const plans = yield* Queue.unbounded(); const storageStarted = yield* Deferred.make(); const storageGate = yield* Deferred.make(); - const service = yield* makeService( - { - launch: makeDefinition(plans).launch, - removeData: () => Effect.void, - }, - { id: "database-storage", config: { version: 17 } }, + const dataRemoved = yield* Ref.make(false); + const service = yield* makeStandaloneService( + { launch: makeDefinition(plans).launch, removeData: () => Ref.set(dataRemoved, true) }, + { id: "database-storage-interrupted", config: { version: 17 } }, ); const storage = yield* service - .storage( - Effect.gen(function* () { - yield* Deferred.succeed(storageStarted, undefined); - yield* Deferred.await(storageGate); - }), - ) + .storage(open(storageStarted).pipe(Effect.andThen(Deferred.await(storageGate)))) .pipe(Effect.forkScoped); yield* Deferred.await(storageStarted); - const destroying = yield* service.destroy.pipe(Effect.forkScoped); - expect((yield* service.get).currentOperation).toBe("storage"); + + const destroyed = yield* service.destroy.pipe(Effect.forkScoped); + yield* awaitDestroyPending(service); + yield* Fiber.interrupt(destroyed); + expect(Exit.hasInterrupts(yield* Fiber.await(destroyed))).toBe(true); + expect((yield* service.get).destroyPending).toBe(false); yield* open(storageGate); yield* Fiber.join(storage); - yield* Fiber.join(destroying); - expect((yield* service.get).registered).toBe(false); + + expect(yield* service.storage(Effect.succeed("stored"))).toBe("stored"); + expect(yield* Ref.get(dataRemoved)).toBe(false); + expect((yield* service.execution).registered).toBe(true); + const plan = yield* makeRuntimePlan; + yield* Queue.offer(plans, plan); + yield* open(plan.launchGate); + yield* service.start; + expect((yield* service.get).lifecycle).toBe("running"); + yield* stopFixture(service, plan); }), ), ); - it.live("prepares invalid restart configuration before stopping current runtime", () => + it.live("prepares an invalid restart configuration before stopping the current runtime", () => Effect.scoped( Effect.gen(function* () { const plans = yield* Queue.unbounded(); - const prepared = yield* Ref.make>([]); const invalid = yield* Ref.make(false); - const service = yield* makeService(makeDefinition(plans, undefined, prepared, invalid), { + const service = yield* makeStandaloneService(makeDefinition(plans, { invalid }), { id: "database-restart", config: { version: 17 }, }); @@ -976,234 +755,86 @@ describe("service kernel", () => { expect(Exit.isFailure(yield* service.restart({ version: 18 }).pipe(Effect.exit))).toBe( true, ); - expect((yield* service.get).lifecycle).toBe("running"); - expect((yield* service.get).config.version).toBe(17); + expect(yield* service.get).toMatchObject({ lifecycle: "running", config: { version: 17 } }); expect((yield* Ref.get(plan.state)).removed).toBe(false); yield* stopFixture(service, plan); }), ), ); - it.live("fails readiness with the crash error and cleans the exact runtime", () => + it.live("fails readiness with the crash error, cleans the exact runtime and stays wakeable", () => Effect.scoped( Effect.gen(function* () { - const fixture = yield* makeFixture; + const fixture = yield* makeFixture("database-crash"); const plan = yield* makeRuntimePlan; yield* Queue.offer(fixture.plans, plan); yield* open(plan.launchGate); yield* fixture.service.start; yield* Deferred.await(plan.healthStarted); - const ready = yield* fixture.service.ready.pipe(Effect.forkScoped); - const stopping = yield* waitForObservation( - fixture.service, - (value) => value.lifecycle === "stopping", + const ready = yield* fixture.service.ready.pipe( + Effect.flip, + Effect.forkScoped({ startImmediately: true }), ); - const stopped = yield* waitForObservation( + const stopped = yield* waitForStatus( fixture.service, - (value) => value.lifecycle === "stopped", + (value) => value.lifecycle === "stopped" && value.currentOperation === undefined, ); yield* Deferred.succeed( plan.exit, Exit.fail(new ServiceError({ operation: "process", message: "crashed" })), ); yield* Deferred.await(plan.stopStarted); + expect((yield* fixture.service.get).currentOperation).toBe("stop"); yield* open(plan.stopGate); yield* Deferred.await(plan.removeStarted); yield* open(plan.removeGate); - yield* Fiber.join(stopping); yield* Fiber.join(stopped); - const result = yield* Fiber.await(ready); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const failure = result.cause.reasons.find(Cause.isFailReason); - expect(failure?.error).toBeInstanceOf(ServiceError); - expect(failure?.error).toMatchObject({ operation: "process", message: "crashed" }); - } - expect((yield* fixture.service.get).lifecycle).toBe("stopped"); + + expect((yield* Fiber.join(ready)).message).toContain("crashed"); + expect(yield* fixture.service.get).toMatchObject({ + lifecycle: "stopped", + wakeEnabled: true, + error: { operation: "process", message: "crashed" }, + }); expect((yield* Ref.get(plan.state)).removed).toBe(true); - expect((yield* fixture.service.get).wakeEnabled).toBe(false); }), ), ); - it.live("reports stop as the current operation during exit cleanup", () => + it.live("fails an old readiness wait on stop and lets the next launch become ready", () => Effect.scoped( Effect.gen(function* () { - const fixture = yield* makeFixture; - const plan = yield* makeRuntimePlan; - yield* Queue.offer(fixture.plans, plan); - yield* open(plan.launchGate); + const fixture = yield* makeFixture("database-relaunch"); + const first = yield* makeRuntimePlan; + const second = yield* makeRuntimePlan; + yield* Queue.offer(fixture.plans, first); + yield* Queue.offer(fixture.plans, second); + yield* open(first.launchGate); yield* fixture.service.start; - yield* Deferred.succeed( - plan.exit, - Exit.fail(new ServiceError({ operation: "process", message: "crashed" })), - ); - yield* Deferred.await(plan.stopStarted); - expect((yield* fixture.service.get).currentOperation).toBe("stop"); - yield* open(plan.stopGate); - yield* Deferred.await(plan.removeStarted); - yield* open(plan.removeGate); - }), - ), - ); + yield* Deferred.await(first.healthStarted); - it.live("preserves cleanupError while a retry is admitted and blocked", () => - Effect.scoped( - Effect.gen(function* () { - const fixture = yield* makeFixture; - const plan = yield* makeRuntimePlan; - yield* Queue.offer(fixture.plans, plan); - yield* open(plan.launchGate); - yield* fixture.service.start; - yield* Ref.set(plan.stopFailure, true); - expect(Exit.isFailure(yield* fixture.service.stop.pipe(Effect.exit))).toBe(true); - expect((yield* fixture.service.get).cleanupError).toBeDefined(); + const oldReady = yield* fixture.service.ready.pipe(Effect.exit, Effect.forkScoped); + const stopping = yield* fixture.service.stop.pipe(Effect.forkScoped); + yield* Deferred.await(first.stopStarted); + yield* open(first.stopGate); + yield* Deferred.await(first.removeStarted); + yield* open(first.removeGate); + yield* Fiber.join(stopping); + expect(Exit.isFailure(yield* Fiber.join(oldReady))).toBe(true); - const retry = yield* fixture.service.stop.pipe(Effect.forkScoped); - yield* Deferred.await(plan.stopRetryStarted); - const exited = yield* waitForObservation( - fixture.service, - (value) => value.exit !== undefined, - ); - yield* Deferred.succeed( - plan.exit, - Exit.fail(new ServiceError({ operation: "process", message: "crashed" })), - ); - yield* Fiber.join(exited); - expect((yield* fixture.service.get).cleanupError).toBeDefined(); - yield* open(plan.stopGate); - yield* Deferred.await(plan.removeStarted); - yield* open(plan.removeGate); - yield* Fiber.join(retry); - expect((yield* fixture.service.get).cleanupError).toBeUndefined(); + yield* open(second.launchGate); + yield* fixture.service.start; + yield* Deferred.await(second.healthStarted); + const newReady = yield* fixture.service.ready.pipe(Effect.forkScoped); + yield* open(second.healthGate); + expect(Exit.isSuccess(yield* Fiber.await(newReady))).toBe(true); + expect((yield* fixture.service.get).launchId).toBe(2); + yield* stopFixture(fixture.service, second); }), ), ); - - it.live( - "makes an old readiness wait stale across stop and lets the new launch become ready", - () => - Effect.scoped( - Effect.gen(function* () { - const fixture = yield* makeFixture; - const first = yield* makeRuntimePlan; - const second = yield* makeRuntimePlan; - yield* Queue.offer(fixture.plans, first); - yield* Queue.offer(fixture.plans, second); - yield* open(first.launchGate); - yield* fixture.service.start; - yield* Deferred.await(first.healthStarted); - - const oldReady = yield* fixture.service.ready.pipe(Effect.forkScoped); - const stopping = yield* fixture.service.stop.pipe(Effect.forkScoped); - yield* Deferred.await(first.stopStarted); - yield* open(first.stopGate); - yield* Deferred.await(first.removeStarted); - yield* open(first.removeGate); - yield* Fiber.join(stopping); - const stale = yield* Fiber.await(oldReady); - expect(Exit.isFailure(stale)).toBe(true); - if (Exit.isFailure(stale)) - expect(stale.cause.reasons.find(Cause.isFailReason)?.error).toBeInstanceOf( - ServiceStaleLaunch, - ); - - yield* open(second.launchGate); - yield* fixture.service.start; - yield* Deferred.await(second.healthStarted); - const newReady = yield* fixture.service.ready.pipe(Effect.forkScoped); - yield* open(second.healthGate); - expect(Exit.isSuccess(yield* Fiber.await(newReady))).toBe(true); - expect((yield* fixture.service.get).launchId).toBe(2); - yield* stopFixture(fixture.service, second); - }), - ), - ); }); -it.live("retains armed wake after a partially launched process exits during cleanup", () => - Effect.scoped( - Effect.gen(function* () { - const exited = yield* Deferred.make>(); - const stopping = yield* Deferred.make(); - const finishStop = yield* Deferred.make(); - const failure = new ServiceError({ operation: "launch", message: "initialization failed" }); - const service = yield* makeService( - { - launch: () => - Effect.fail( - new ServiceLaunchError({ - failure, - runtime: { - health: Effect.never, - exit: Deferred.await(exited), - stop: Deferred.succeed(stopping, undefined).pipe( - Effect.andThen(Deferred.await(finishStop)), - ), - remove: Effect.void, - }, - }), - ), - removeData: () => Effect.void, - }, - { id: "armed-partial", config: {} }, - ); - yield* service.arm; - const observedExit = yield* waitForObservation(service, (state) => state.exit !== undefined); - const starting = yield* service.start.pipe(Effect.forkChild); - yield* Deferred.await(stopping); - yield* Deferred.succeed(exited, Exit.void); - yield* Fiber.join(observedExit); - yield* Deferred.succeed(finishStop, undefined); - expect(Exit.isFailure(yield* Fiber.await(starting))).toBe(true); - expect((yield* service.get).lifecycle).toBe("stopped"); - expect((yield* service.get).wakeEnabled).toBe(true); - }), - ), -); - -it.live("keeps a sleeping instance armed when exit observation retries failed removal", () => - Effect.scoped( - Effect.gen(function* () { - const exited = yield* Deferred.make>(); - const failRemoval = yield* Ref.make(true); - const service = yield* makeService( - { - launch: () => - Effect.succeed({ - health: Effect.void, - exit: Deferred.await(exited), - stop: Effect.void, - remove: Ref.getAndSet(failRemoval, false).pipe( - Effect.flatMap((fail) => - fail - ? Effect.fail( - new ServiceError({ - operation: "remove", - message: "temporary removal failure", - }), - ) - : Effect.void, - ), - ), - }), - removeData: () => Effect.void, - }, - { id: "sleep-cleanup-retry", config: {} }, - ); - yield* service.arm; - yield* service.start; - yield* service.ready; - expect(Exit.isFailure(yield* Effect.exit(service.sleep))).toBe(true); - expect((yield* service.get).wakeEnabled).toBe(true); - const stopped = yield* waitForObservation(service, (state) => state.lifecycle === "stopped"); - yield* Deferred.succeed(exited, Exit.void); - yield* Fiber.join(stopped); - expect((yield* service.get).wakeEnabled).toBe(true); - }), - ), -); - const unhealthy = (message: string) => new ServiceError({ operation: "health", message }); const makeProbedService = ( @@ -1212,7 +843,7 @@ const makeProbedService = ( ) => Effect.gen(function* () { const launches = yield* Ref.make(0); - const service = yield* makeService( + const service = yield* makeStandaloneService( { launch: () => Ref.updateAndGet(launches, (count) => count + 1).pipe( @@ -1231,9 +862,9 @@ const makeProbedService = ( return { service, launches }; }); -const startUnhealthy = (service: ServiceInstance) => +const startUnhealthy = (service: Standalone) => Effect.gen(function* () { - const failed = yield* waitForObservation(service, (value) => value.health === "unhealthy"); + const failed = yield* waitForStatus(service, (value) => value.health === "unhealthy"); yield* service.start; yield* Fiber.join(failed); }); @@ -1249,7 +880,7 @@ describe("service readiness recovery", () => { ), ); yield* startUnhealthy(service); - expect(yield* Effect.flip(service.ready)).toMatchObject({ message: "still booting" }); + expect((yield* Effect.flip(service.ready)).message).toContain("still booting"); yield* Ref.set(healthy, true); yield* service.ready; @@ -1292,9 +923,7 @@ describe("service readiness recovery", () => { yield* open(checkGate); - expect(yield* Fiber.join(waiting)).toMatchObject({ - message: "rest HTTP readiness timed out", - }); + expect((yield* Fiber.join(waiting)).message).toContain("rest HTTP readiness timed out"); expect(yield* Ref.get(checks)).toBe(1); expect(yield* service.get).toMatchObject({ health: "unhealthy", @@ -1320,8 +949,8 @@ describe("service readiness recovery", () => { ); yield* startUnhealthy(service); - expect(yield* Effect.flip(service.ready)).toMatchObject({ message: "setup failed" }); - expect(yield* Effect.flip(service.ready)).toMatchObject({ message: "setup failed" }); + expect((yield* Effect.flip(service.ready)).message).toContain("setup failed"); + expect((yield* Effect.flip(service.ready)).message).toContain("setup failed"); expect(yield* Ref.get(checks)).toBe(1); yield* service.stop; }), @@ -1359,8 +988,8 @@ describe("service readiness recovery", () => { yield* Deferred.await(probeStarted); yield* open(probeGate); - expect(yield* Fiber.join(first)).toMatchObject({ message: "still booting" }); - expect(yield* Fiber.join(second)).toMatchObject({ message: "still booting" }); + expect((yield* Fiber.join(first)).message).toContain("still booting"); + expect((yield* Fiber.join(second)).message).toContain("still booting"); expect(yield* Ref.get(checks)).toBe(2); yield* service.ready; expect(yield* Ref.get(checks)).toBe(3); @@ -1390,10 +1019,12 @@ describe("service readiness recovery", () => { : Effect.fail(unhealthy("second launch unhealthy")), ); yield* startUnhealthy(service); - const stale = yield* Effect.forkChild(service.ready, { startImmediately: true }); + const stale = yield* Effect.forkChild(Effect.exit(service.ready), { + startImmediately: true, + }); yield* Deferred.await(probeStarted); - const relaunched = yield* waitForObservation( + const relaunched = yield* waitForStatus( service, (value) => value.launchId === 2 && value.health === "unhealthy", ); @@ -1401,15 +1032,13 @@ describe("service readiness recovery", () => { yield* Fiber.join(relaunched); yield* Deferred.await(probeInterrupted); - expect(Exit.isFailure(yield* Fiber.await(stale))).toBe(true); + expect(Exit.isFailure(yield* Fiber.join(stale))).toBe(true); expect(yield* service.get).toMatchObject({ launchId: 2, health: "unhealthy", error: { message: "second launch unhealthy" }, }); - expect(yield* Effect.flip(service.ready)).toMatchObject({ - message: "second launch unhealthy", - }); + expect((yield* Effect.flip(service.ready)).message).toContain("second launch unhealthy"); expect(yield* Ref.get(firstLaunchChecks)).toBe(2); yield* service.stop; }), @@ -1439,7 +1068,7 @@ describe("service readiness recovery", () => { yield* Scope.close(owner, Exit.void); - expect(yield* Fiber.join(waiting)).toMatchObject({ message: "Session stopped" }); + expect((yield* Fiber.join(waiting)).message).toBe("Lifecycle owner stopped"); }), ), ); diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index 58ddfb6ace..6a7d3c899e 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -1,7 +1,6 @@ import { Cause, Data, - Deferred, Effect, Exit, Fiber, @@ -12,24 +11,22 @@ import { SubscriptionRef, } from "effect"; import type { Stream } from "effect"; +import { LifecycleEvent } from "./Lifecycle.ts"; -type ServiceLifecycle = "stopped" | "starting" | "running" | "stopping"; -type ServiceHealth = "starting" | "healthy" | "unhealthy"; +/** The execution step a service is performing right now, for status reporting. */ type ServiceOperation = "start" | "stop" | "restart" | "storage" | "destroy" | "sleep"; -export type ServiceAdmission = ServiceOperation | "arm"; +/** + * Execution facts about one service: its configuration and the outcome of its last launch, check, + * exit or cleanup. Lifecycle phase and health come from the lifecycle reducer, not from here. + */ export interface ServiceObservation { readonly id: string; readonly config: Config; - readonly lifecycle: ServiceLifecycle; - readonly health: ServiceHealth | undefined; readonly error: ServiceError | undefined; readonly cleanupError: ServiceError | undefined; readonly exit: Exit.Exit | undefined; readonly currentOperation: ServiceOperation | undefined; - readonly launchId: number | undefined; - readonly intentRevision: number; - readonly wakeEnabled: boolean; readonly registered: boolean; } @@ -39,26 +36,6 @@ export class ServiceError extends Data.TaggedError("ServiceError")<{ readonly cause?: unknown; }> {} -export class ServiceDestroyed extends Data.TaggedError("ServiceDestroyed")<{ - readonly id: string; -}> {} - -export class ServiceNotRunning extends Data.TaggedError("ServiceNotRunning")<{ - readonly id: string; -}> {} - -class ServiceNotStopped extends Data.TaggedError("ServiceNotStopped")<{ - readonly id: string; - readonly lifecycle: ServiceLifecycle; - readonly message: string; -}> {} - -export class ServiceStaleLaunch extends Data.TaggedError("ServiceStaleLaunch")<{ - readonly id: string; - readonly launchId: number; - readonly message: string; -}> {} - /** The exact runtime resources owned by one service launch. */ export interface RuntimeSession { /** The initial readiness program for this session. */ @@ -80,7 +57,7 @@ export class ServiceLaunchError extends Data.TaggedError("ServiceLaunchError")<{ }> {} export interface ServiceDefinition { - /** Preparation is performed before lifecycle admission. */ + /** Idempotent preparation (artifacts, images) run at the start of every launch. */ readonly prepare?: (config: Config) => Effect.Effect; /** Failures after resource acquisition carry the session for ordinary cleanup. */ readonly launch: ( @@ -98,73 +75,51 @@ export interface ServiceInstanceContext { readonly scope: Scope.Closeable; } +/** + * The execution boundary for one service: it runs the launches, checks and stops the lifecycle + * reducer commands for a generation, holds that generation's resources until their cleanup is + * confirmed, and reports every outcome back as a generation-tagged event. + */ export interface ServiceInstance { readonly id: string; readonly observation: Stream.Stream>; readonly get: Effect.Effect>; - readonly start: Effect.Effect; - readonly startAt: ( - revision: number, - config?: Config, - wake?: boolean, - guard?: Effect.Effect, - ) => Effect.Effect; - readonly arm: Effect.Effect; - readonly armAt: ( - revision: number, - guard?: Effect.Effect, - ) => Effect.Effect; - readonly sleep: Effect.Effect; - readonly stop: Effect.Effect; - readonly restart: ( - config?: Config, - revision?: number, - guard?: Effect.Effect, - ) => Effect.Effect; - readonly ready: Effect.Effect< - void, - ServiceError | ServiceDestroyed | ServiceNotRunning | ServiceStaleLaunch - >; - /** Runs instance-owned storage work only while the instance is stopped. */ + readonly prepare: (config: Config) => Effect.Effect; + /** Runs generation `generation`'s launch and readiness check; outcomes are reported, never returned. */ + readonly launch: (generation: number, config: Config) => Effect.Effect; + /** + * Stops `generation`'s session, never a newer one, and reports `Exited` once cleanup is + * confirmed or `StopFailed` while its resources are still held. + */ + readonly stop: ( + generation: number, + options: { readonly operation: ServiceOperation; readonly discard: boolean }, + ) => Effect.Effect; + /** Re-runs a failed readiness check of `generation`'s live session and reports the result. */ + readonly reprobe: (generation: number) => Effect.Effect; + /** Runs instance-owned storage work; the lifecycle reservation keeps the service stopped. */ readonly storage: ( operation: Effect.Effect, - ) => Effect.Effect; - readonly destroy: Effect.Effect; + ) => Effect.Effect; + /** Removes the instance's data and marks it unregistered; the service must already be stopped. */ + readonly removeData: Effect.Effect; } -type HealthCheck = Deferred.Deferred>; - interface SessionRecord { - readonly launchId: number; + readonly generation: number; readonly runtime: RuntimeSession; + /** The execution handle's scope, closed only after the runtime's stop and removal succeed. */ readonly scope: Scope.Closeable; readonly healthScope: Scope.Closeable; - /** The latest readiness check of this launch; a settled failure is replaced by an on-demand probe. */ - readonly health: Ref.Ref; readonly stopped: Ref.Ref; readonly removed: Ref.Ref; } -/** - * Owns one launch attempt's resources from before preparation through confirmed cleanup. A stale - * or abandoned attempt still closes its scope; a successful one hands that duty to `stopNow`. - */ -interface ExecutionHandle { - readonly scope: Scope.Closeable; +interface Attempt { + readonly generation: number; + readonly fiber: Fiber.Fiber; } -const sessionStopped = () => new ServiceError({ operation: "health", message: "Session stopped" }); - -const contextFor = ( - id: string, - config: Config, - scope: Scope.Closeable, -): ServiceInstanceContext => ({ - id, - config, - scope, -}); - const exitError = ( operation: string, exit: Exit.Exit, @@ -181,647 +136,373 @@ const exitError = ( }), ); -/** Creates one independently serialized service instance. */ +const destroyedError = (id: string) => + new ServiceError({ operation: "launch", message: `Service ${id} was destroyed` }); + +/** Creates one service's execution boundary; launches and cleanups run in the current scope. */ export const makeService = ( definition: ServiceDefinition, options: { readonly id: string; readonly config: Config; - readonly coordinate?: ( - operation: ServiceAdmission, - transition: Effect.Effect, - ) => Effect.Effect; + /** Delivers an execution outcome to the lifecycle authority. */ + readonly report: (event: LifecycleEvent) => Effect.Effect; }, ): Effect.Effect, never, Scope.Scope> => Effect.gen(function* () { const owner = yield* Scope.Scope; - const gate = yield* Semaphore.make(1); - const revision = yield* Ref.make(0); - const launchCounter = yield* Ref.make(0); + const { id, report } = options; + // Serializes every step that touches runtime resources: a launch's acquisition, a stop, an + // exit's cleanup, storage work and data removal. + const execution = yield* Semaphore.make(1); const current = yield* Ref.make(undefined); - // Forked before preparation starts so an abandoned or stale attempt still has a scope to close. - const allocateHandle: Effect.Effect = Scope.fork(owner, "parallel").pipe( - Effect.map((scope): ExecutionHandle => ({ scope })), - ); - // stopNow clears observation.launchId while keeping the error, so readiness must not read that field. - const launchError = yield* Ref.make< - { readonly launchId: number; readonly error: ServiceError } | undefined - >(undefined); + const attempt = yield* Ref.make(undefined); + // The highest generation already stopped: a launch for it, admitted late, must never run. + const stoppedThrough = yield* Ref.make(0); const config = yield* Ref.make(options.config); const observations = yield* SubscriptionRef.make>({ - id: options.id, + id, config: options.config, - lifecycle: "stopped", - health: undefined, error: undefined, cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, - intentRevision: 0, - wakeEnabled: false, registered: true, }); const update = (change: Partial>) => SubscriptionRef.update(observations, (value) => ({ ...value, ...change })); + const recordCleanupError = (error: ServiceError) => + SubscriptionRef.update(observations, (value) => ({ + ...value, + error: value.error ?? error, + cleanupError: error, + })); + + /** Halts and removes one session, retrying only the steps a previous attempt didn't finish. */ + const stopRecord = Effect.fn("Service.stopRecord")(function* ( + record: SessionRecord, + discard: boolean, + ) { + yield* Effect.annotateCurrentSpan({ member_id: id, generation: record.generation }); + yield* Scope.close(record.healthScope, Exit.void); + if (!(yield* Ref.get(record.stopped))) { + const halt = + discard && record.runtime.discard !== undefined + ? record.runtime.discard + : record.runtime.stop; + yield* halt.pipe(Effect.tapError(recordCleanupError)); + yield* Ref.set(record.stopped, true); + } + if (!(yield* Ref.get(record.removed))) { + yield* record.runtime.remove.pipe(Effect.tapError(recordCleanupError)); + yield* Ref.set(record.removed, true); + } + yield* Scope.close(record.scope, Exit.void); + yield* Ref.update(current, (value) => (value === record ? undefined : value)); + yield* update({ cleanupError: undefined }); + }); - const setOperation = (operation: ServiceOperation | undefined) => - update({ currentOperation: operation }); - const coordinate = - options.coordinate ?? - ((_operation: ServiceAdmission, transition: Effect.Effect) => transition); - - // Health and exit settlement each write the observation and launchError as one step. - const settlement = yield* Semaphore.make(1); + /** Runs one readiness program in the session's health scope; interruption yields `undefined`. */ + const check = (record: SessionRecord, program: Effect.Effect) => + Effect.forkIn(program, record.healthScope, { startImmediately: true }).pipe( + Effect.flatMap(Fiber.await), + Effect.map((exit) => + Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause) ? undefined : exit, + ), + ); const settleCheck = Effect.fn("Service.settleCheck")(function* ( record: SessionRecord, exit: Exit.Exit, + recovered: boolean, ) { const error = exitError("health", exit); - yield* settlement.withPermit( - Effect.gen(function* () { - const owned = yield* SubscriptionRef.modify( - observations, - (observation): [boolean, ServiceObservation] => - observation.launchId === record.launchId && observation.lifecycle === "running" - ? [ - true, - { - ...observation, - health: error === undefined ? "healthy" : "unhealthy", - error, - }, - ] - : [false, observation], - ); - if (owned) - yield* Ref.set( - launchError, - error === undefined ? undefined : { launchId: record.launchId, error }, - ); - }), + yield* Effect.annotateCurrentSpan({ healthy: error === undefined }); + if ((yield* Ref.get(current)) !== record) return; + yield* update({ error }); + const generation = record.generation; + yield* report( + error !== undefined + ? LifecycleEvent.ReadinessLost({ id, generation, cause: error }) + : recovered + ? LifecycleEvent.ReadinessRecovered({ id, generation }) + : LifecycleEvent.LaunchSucceeded({ id, generation }), ); }); - /** Runs one readiness check whose result is shared by every `ready` caller awaiting it. */ - const runCheck = Effect.fn("Service.runCheck")( - (record: SessionRecord, check: Effect.Effect, result: HealthCheck) => - Effect.forkIn( - check.pipe( - Effect.onExit((exit) => - Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause) - ? Deferred.succeed(result, Exit.fail(sessionStopped())) - : settleCheck(record, exit).pipe(Effect.andThen(Deferred.succeed(result, exit))), - ), - ), - record.healthScope, - // Starting immediately installs the settlement before a closing scope can interrupt it. - { startImmediately: true, uninterruptible: false }, - ), - ); - - // Mask only the permit handoff; admitted work belongs to the host scope. `onAdmitted` runs - // uninterruptibly right after the fork succeeds, so a caller can transfer resource-cleanup - // ownership to the owner fiber atomically: once it has run, the caller's own interruption of - // the join below can no longer race ahead of that transfer. - const run = Effect.fn("Service.run")(function* < - A, - E extends ServiceError | ServiceDestroyed | ServiceNotStopped, - >(operation: Effect.Effect, onAdmitted: Effect.Effect = Effect.void) { - return yield* Effect.uninterruptibleMask((restore) => + /** Cleans a session whose runtime exited on its own, then reports the exit. */ + const observeExit = Effect.fn("Service.observeExit")(function* ( + record: SessionRecord, + exit: Exit.Exit, + ) { + yield* Effect.annotateCurrentSpan({ member_id: id, generation: record.generation }); + const error = + exitError("exit", exit) ?? + new ServiceError({ operation: "exit", message: "Runtime exited unexpectedly" }); + const cleaned = yield* execution.withPermit( Effect.gen(function* () { - yield* restore(gate.take(1)); - const fiber = yield* Effect.forkIn( - operation.pipe(Effect.ensuring(gate.release(1))), - owner, - { uninterruptible: false }, + if ((yield* Ref.get(current)) !== record) return undefined; + yield* update({ error, exit, currentOperation: "stop" }); + return yield* stopRecord(record, false).pipe( + Effect.exit, + Effect.ensuring(update({ currentOperation: undefined })), ); - yield* onAdmitted; - return yield* restore(Fiber.join(fiber)); }), ); + if (cleaned === undefined) return; + const generation = record.generation; + yield* report( + Exit.isSuccess(cleaned) + ? LifecycleEvent.Exited({ id, generation, cause: error, requested: false }) + : LifecycleEvent.StopFailed({ + id, + generation, + cause: Cause.squash(cleaned.cause), + failure: error, + }), + ); }); - const stopNow = Effect.fn("Service.stopNow")(function* ( - expected?: SessionRecord, - retainWake = false, - discard = false, - ) { - yield* Effect.annotateCurrentSpan({ member_id: options.id }); - yield* Effect.gen(function* () { - const record = yield* Ref.get(current); - if (expected !== undefined && record !== expected) return; - if (record === undefined) { - yield* update({ lifecycle: "stopped", health: undefined }); - return; - } - - yield* update({ lifecycle: "stopping", health: undefined, wakeEnabled: retainWake }); - yield* Deferred.succeed(yield* Ref.get(record.health), Exit.fail(sessionStopped())); - yield* Scope.close(record.healthScope, Exit.void); - if (!(yield* Ref.get(record.stopped))) { - const halt = - discard && record.runtime.discard !== undefined - ? record.runtime.discard - : record.runtime.stop; - yield* halt.pipe( - Effect.tapError((error) => - SubscriptionRef.update(observations, (value) => ({ - ...value, - error: value.error ?? error, - cleanupError: error, - })), - ), - ); - yield* Ref.set(record.stopped, true); - } - if (!(yield* Ref.get(record.removed))) { - yield* record.runtime.remove.pipe( - Effect.tapError((error) => - SubscriptionRef.update(observations, (value) => ({ - ...value, - error: value.error ?? error, - cleanupError: error, - })), - ), - ); - yield* Ref.set(record.removed, true); - } - yield* Scope.close(record.scope, Exit.void); - - yield* Ref.set(current, undefined); - yield* update({ - lifecycle: "stopped", - health: undefined, - launchId: undefined, - cleanupError: undefined, - }); - }); - }); - - const launchNow = Effect.fn("Service.launchNow")(function* ( - handle: ExecutionHandle, - handedOff: Ref.Ref, - guard: Effect.Effect, + /** Acquires one generation's runtime; on failure, its retained resources are cleaned first. */ + const acquire = Effect.fn("Service.acquire")(function* ( + generation: number, + launchConfig: Config, + handle: Scope.Closeable, ) { - yield* Effect.annotateCurrentSpan({ member_id: options.id }); - yield* Effect.gen(function* () { - let observation = yield* SubscriptionRef.get(observations); - if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); - if (observation.lifecycle === "running" || observation.lifecycle === "starting") return; - if (observation.lifecycle === "stopping") { - yield* stopNow(undefined, observation.wakeEnabled); - observation = yield* SubscriptionRef.get(observations); - if (observation.lifecycle !== "stopped") return; - } - - const launchId = yield* Ref.updateAndGet(launchCounter, (value) => value + 1); - yield* coordinate( - "start", - guard.pipe( - Effect.andThen( - Ref.set(launchError, undefined).pipe( - Effect.andThen( - update({ - lifecycle: "starting", - health: "starting", - error: undefined, - cleanupError: undefined, - exit: undefined, - wakeEnabled: observation.wakeEnabled, - launchId, - }), - ), - ), - ), + if (generation <= (yield* Ref.get(stoppedThrough))) return { _tag: "Fenced" } as const; + const observation = yield* SubscriptionRef.get(observations); + if (!observation.registered) return { _tag: "Failed", error: destroyedError(id) } as const; + // The lifecycle launches only once the previous session's cleanup is confirmed. + if ((yield* Ref.get(current)) !== undefined) + return { + _tag: "Failed", + error: new ServiceError({ + operation: "launch", + message: `Service ${id} still holds a previous session`, + }), + } as const; + yield* Ref.set(config, launchConfig); + yield* update({ config: launchConfig, exit: undefined, currentOperation: "start" }); + const launched = yield* Effect.exit( + definition.launch({ id, config: launchConfig, scope: handle }).pipe( + Effect.map((runtime) => ({ runtime, failure: undefined })), + Effect.catchTag("ServiceLaunchError", ({ runtime, failure }) => + Effect.succeed({ runtime, failure }), ), - ); - const runtimeScope = handle.scope; - const launchExit = yield* Effect.exit( - definition.launch(contextFor(options.id, yield* Ref.get(config), runtimeScope)).pipe( - Effect.map((runtime) => ({ runtime, failure: undefined })), - Effect.catchTag("ServiceLaunchError", ({ runtime, failure }) => - Effect.succeed({ runtime, failure }), + ), + ); + if (Exit.isFailure(launched)) { + yield* Scope.close(handle, launched); + return { + _tag: "Failed", + error: + exitError("launch", launched) ?? + new ServiceError({ operation: "launch", message: "Launch failed" }), + } as const; + } + const { runtime, failure } = launched.value; + const record: SessionRecord = { + generation, + runtime, + scope: handle, + healthScope: yield* Scope.fork(handle, "parallel"), + stopped: yield* Ref.make(false), + removed: yield* Ref.make(false), + }; + yield* Ref.set(current, record); + if (failure !== undefined) { + const cleaned = yield* stopRecord(record, false).pipe(Effect.result); + return cleaned._tag === "Failure" + ? ({ _tag: "Retained", error: failure, cleanup: cleaned.failure } as const) + : ({ _tag: "Failed", error: failure } as const); + } + yield* Effect.forkIn( + runtime.exit.pipe( + // Recorded at once, so a requested stop's exit is visible when the stop completes. + Effect.tap((exit) => + Ref.get(current).pipe( + Effect.flatMap((live) => (live === record ? update({ exit }) : Effect.void)), ), ), - ); - if (Exit.isFailure(launchExit)) { - yield* Scope.close(runtimeScope, launchExit); - const error = exitError("launch", launchExit); - if (error !== undefined) yield* Ref.set(launchError, { launchId, error }); - yield* update({ - lifecycle: "stopped", - health: undefined, - wakeEnabled: observation.wakeEnabled, - error, - launchId: undefined, - }); - return yield* Effect.failCause(launchExit.cause); - } - const { runtime, failure: launchFailure } = launchExit.value; - const health = yield* Deferred.make>(); - const record: SessionRecord = { - launchId, - runtime, - scope: runtimeScope, - healthScope: yield* Scope.fork(runtimeScope, "parallel"), - health: yield* Ref.make(health), - stopped: yield* Ref.make(false), - removed: yield* Ref.make(false), - }; - yield* Ref.set(current, record); - // From here, record.scope (== handle.scope) is stopNow's responsibility, not the caller's. - yield* Ref.set(handedOff, true); - if (launchFailure !== undefined) - yield* Ref.set(launchError, { launchId, error: launchFailure }); - yield* update({ - lifecycle: launchFailure === undefined ? "running" : "stopping", - health: launchFailure === undefined ? "starting" : undefined, - error: launchFailure, - launchId, - config: yield* Ref.get(config), - }); - - const observeExit = record.runtime.exit.pipe( + // Cleanup closes this session's scope, so it runs on the owner's fiber instead. Effect.flatMap((exit) => - Effect.gen(function* () { - const owned = yield* settlement.withPermit( - Effect.gen(function* () { - if ((yield* Ref.get(current)) !== record) return false; - const observation = yield* SubscriptionRef.get(observations); - const error = - observation.lifecycle === "stopping" - ? observation.error - : (exitError("exit", exit) ?? - new ServiceError({ - operation: "exit", - message: "Runtime exited unexpectedly", - })); - if (error !== undefined) - yield* Ref.set(launchError, { launchId: record.launchId, error }); - yield* update({ - lifecycle: "stopping", - health: undefined, - error, - exit, - wakeEnabled: observation.lifecycle === "stopping" && observation.wakeEnabled, - }); - return true; - }), - ); - if (!owned) return; - yield* run( - Effect.gen(function* () { - if ((yield* Ref.get(current)) !== record) return; - yield* setOperation("stop"); - const live = yield* SubscriptionRef.get(observations); - yield* stopNow(record, live.wakeEnabled).pipe( - Effect.ensuring(setOperation(undefined)), - ); - }), - ); - }), + Effect.forkIn(observeExit(record, exit), owner, { startImmediately: true }), ), - ); - if (launchFailure === undefined) yield* runCheck(record, runtime.health, health); - yield* Effect.forkIn(observeExit, runtimeScope, { uninterruptible: false }); - if (launchFailure !== undefined) { - yield* Deferred.succeed(health, Exit.fail(launchFailure)); - yield* stopNow(undefined, observation.wakeEnabled); - return yield* launchFailure; - } - }); - }); - - /** A wake has no caller to receive a preparation failure, so an idle observation records it. */ - const recordPreparationFailure = Effect.fn("Service.recordPreparationFailure")(function* ( - expectedRevision: number, - error: ServiceError, - ) { - if ((yield* Ref.get(revision)) !== expectedRevision) return; - yield* SubscriptionRef.update(observations, (value) => - value.registered && value.lifecycle === "stopped" ? { ...value, error } : value, + ), + handle, + { startImmediately: true }, ); + return { _tag: "Live", record } as const; }); - const startAt = Effect.fn("Service.startAt")(function* ( - expectedRevision: number, - candidate?: Config, - wake = false, - guard: Effect.Effect = Effect.void, + const run = Effect.fn("Service.launch")(function* ( + generation: number, + launchConfig: Config, + handle: Scope.Closeable, ) { - yield* Effect.annotateCurrentSpan({ member_id: options.id }); - let existing = yield* SubscriptionRef.get(observations); - if (!existing.registered) return yield* new ServiceDestroyed({ id: options.id }); - if (existing.lifecycle === "running") return; - if (existing.lifecycle === "starting") { - yield* run(Effect.void); - existing = yield* SubscriptionRef.get(observations); - if (!existing.registered) return yield* new ServiceDestroyed({ id: options.id }); - if (existing.lifecycle === "running") return; + yield* Effect.annotateCurrentSpan({ member_id: id, generation }); + yield* update({ error: undefined }); + const prepared = yield* (definition.prepare?.(launchConfig) ?? Effect.void).pipe(Effect.exit); + if (Exit.isFailure(prepared)) { + // A stopped generation's late failure must not mark the observation of its successor. + if (generation <= (yield* Ref.get(stoppedThrough))) return; + const error = exitError("prepare", prepared); + yield* update({ error }); + yield* report(LifecycleEvent.LaunchFailed({ id, generation, cause: error })); + return; } - const nextConfig = candidate ?? (yield* Ref.get(config)); - const handle = yield* allocateHandle; - const handedOff = yield* Ref.make(false); - // Set once `run` admits the operation onto the owner fiber; from that point the owner alone - // decides the handle's fate (via the inner `ensuring` below), atomically with the fork, so - // the caller's own interruption of the join can never race ahead of that decision. - const admitted = yield* Ref.make(false); - yield* Effect.gen(function* () { - if (definition.prepare !== undefined) - yield* definition - .prepare(nextConfig) - .pipe(Effect.tapError((error) => recordPreparationFailure(expectedRevision, error))); - yield* run( - Effect.gen(function* () { - const observation = yield* SubscriptionRef.get(observations); - if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); - if (observation.lifecycle === "running" || observation.lifecycle === "starting") return; - if ( - (yield* Ref.get(revision)) !== expectedRevision || - (wake && !observation.wakeEnabled) - ) - return yield* new ServiceError({ - operation: "admission", - message: "Service intent changed before admission", - }); - yield* setOperation("start"); - yield* launchNow( - handle, - handedOff, - guard.pipe( - Effect.andThen(Ref.set(config, nextConfig)), - Effect.andThen(update({ config: nextConfig })), - ), - ).pipe(Effect.ensuring(setOperation(undefined))); - }).pipe( - Effect.ensuring( - Ref.get(handedOff).pipe( - Effect.flatMap((isHandedOff) => - isHandedOff ? Effect.void : Scope.close(handle.scope, Exit.void), - ), - ), - ), - ), - Ref.set(admitted, true), - ); - }).pipe( - Effect.ensuring( - Ref.get(admitted).pipe( - Effect.flatMap((isAdmitted) => - isAdmitted ? Effect.void : Scope.close(handle.scope, Exit.void), - ), - ), + yield* report(LifecycleEvent.StageChanged({ id, generation, stage: "launching" })); + const acquired = yield* execution.withPermit( + acquire(generation, launchConfig, handle).pipe( + Effect.ensuring(update({ currentOperation: undefined })), ), ); - }); - const start = Ref.get(revision).pipe(Effect.flatMap((revision) => startAt(revision))); - - const invalidate = Effect.fn("Service.invalidate")(function* () { - const next = yield* Ref.updateAndGet(revision, (value) => value + 1); - const record = yield* Ref.get(current); - yield* update({ - intentRevision: next, - wakeEnabled: false, - lifecycle: record === undefined ? "stopped" : "stopping", - health: undefined, - }); + if (acquired._tag === "Fenced") return; + if (acquired._tag === "Retained") { + yield* update({ error: acquired.error }); + yield* report( + LifecycleEvent.StopFailed({ + id, + generation, + cause: acquired.cleanup, + failure: acquired.error, + }), + ); + return; + } + if (acquired._tag === "Failed") { + yield* update({ error: acquired.error }); + yield* report(LifecycleEvent.LaunchFailed({ id, generation, cause: acquired.error })); + return; + } + yield* report(LifecycleEvent.SessionAvailable({ id, generation })); + const healthy = yield* check(acquired.record, acquired.record.runtime.health); + if (healthy !== undefined) yield* settleCheck(acquired.record, healthy, false); }); - const armAt = Effect.fn("Service.armAt")(function* ( - expectedRevision: number, - guard: Effect.Effect = Effect.void, + const launch = Effect.fn("Service.launchAttempt")(function* ( + generation: number, + launchConfig: Config, ) { - yield* Effect.annotateCurrentSpan({ member_id: options.id }); - yield* run( - Effect.gen(function* () { - if (!(yield* SubscriptionRef.get(observations)).registered) - return yield* new ServiceDestroyed({ id: options.id }); - if ((yield* Ref.get(revision)) !== expectedRevision) - return yield* new ServiceError({ - operation: "admission", - message: "Service intent changed before admission", - }); - yield* coordinate("arm", guard.pipe(Effect.andThen(update({ wakeEnabled: true })))); - }), - ); - }); - const arm = Ref.get(revision).pipe(Effect.flatMap((revision) => armAt(revision))); - - const sleep = Effect.fn("Service.sleep")(function* () { - yield* run( - Effect.gen(function* () { - const observation = yield* SubscriptionRef.get(observations); - if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); - if (observation.lifecycle !== "running" || !observation.wakeEnabled) return; - yield* coordinate( - "sleep", - update({ lifecycle: "stopping", health: undefined, currentOperation: "sleep" }), - ); - yield* stopNow(undefined, true).pipe(Effect.ensuring(setOperation(undefined))); - }), - ); - }); - - const stop = Effect.fn("Service.stop")(function* () { - yield* Effect.annotateCurrentSpan({ member_id: options.id }); - yield* run( - Effect.gen(function* () { - const observation = yield* SubscriptionRef.get(observations); - if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); - yield* coordinate("stop", invalidate()); - yield* setOperation("stop"); - yield* stopNow().pipe(Effect.ensuring(setOperation(undefined))); - }), + // Created before preparation so a stopped or abandoned attempt still has a scope to close; + // once a session holds it, only `stopRecord` closes it. + const handle = yield* Scope.fork(owner, "parallel"); + const releaseUnlessHeld = Ref.get(current).pipe( + Effect.flatMap((record) => + record?.scope === handle ? Effect.void : Scope.close(handle, Exit.void), + ), ); - }); - - const restart = Effect.fn("Service.restart")(function* ( - candidate?: Config, - requestedRevision?: number, - guard: Effect.Effect = Effect.void, - ) { - yield* Effect.annotateCurrentSpan({ member_id: options.id }); - const expectedRevision = requestedRevision ?? (yield* Ref.get(revision)); - const nextConfig = candidate ?? (yield* Ref.get(config)); - const handle = yield* allocateHandle; - const handedOff = yield* Ref.make(false); - // See `startAt`: ownership transfers atomically to the owner fiber on admission, so the - // caller's own interruption of the join can never race ahead of that transfer. - const admitted = yield* Ref.make(false); - yield* Effect.gen(function* () { - if (definition.prepare !== undefined) yield* definition.prepare(nextConfig); - yield* run( - Effect.gen(function* () { - const observation = yield* SubscriptionRef.get(observations); - if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); - if ((yield* Ref.get(revision)) !== expectedRevision) - return yield* new ServiceError({ - operation: "admission", - message: "Service intent changed before admission", - }); - yield* coordinate("restart", invalidate()); - yield* setOperation("restart"); - yield* Effect.gen(function* () { - yield* stopNow(); - yield* Ref.set(config, nextConfig); - yield* update({ config: nextConfig }); - yield* launchNow(handle, handedOff, guard); - }).pipe(Effect.ensuring(setOperation(undefined))); - }).pipe( - Effect.ensuring( - Ref.get(handedOff).pipe( - Effect.flatMap((isHandedOff) => - isHandedOff ? Effect.void : Scope.close(handle.scope, Exit.void), - ), - ), - ), + const fiber = yield* Effect.forkIn( + run(generation, launchConfig, handle).pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.void + : report(LifecycleEvent.LaunchFailed({ id, generation, cause: Cause.squash(cause) })), ), - Ref.set(admitted, true), - ); - }).pipe( - Effect.ensuring( - Ref.get(admitted).pipe( - Effect.flatMap((isAdmitted) => - isAdmitted ? Effect.void : Scope.close(handle.scope, Exit.void), - ), + Effect.ensuring(releaseUnlessHeld), + Effect.ensuring( + Ref.update(attempt, (value) => (value?.generation === generation ? undefined : value)), ), ), + owner, + { startImmediately: true, uninterruptible: false }, ); + yield* Ref.set(attempt, { generation, fiber }); + yield* Fiber.await(fiber); }); - const staleLaunch = (launchId: number, message: string) => - new ServiceStaleLaunch({ id: options.id, launchId, message }); - const diedBeforeReady = (launchId: number) => - Effect.gen(function* () { - const owned = yield* Ref.get(launchError); - if (owned !== undefined && owned.launchId === launchId) return yield* owned.error; - return yield* staleLaunch(launchId, `Service ${options.id} stopped before it was ready`); - }); - const isLive = (record: SessionRecord) => - Effect.gen(function* () { - if ((yield* Ref.get(current)) !== record) return false; - return (yield* SubscriptionRef.get(observations)).lifecycle === "running"; - }); - - /** Concurrent callers share one probe per settled failure of a launch. */ - const reprobe = Effect.fn("Service.reprobe")(function* ( - record: SessionRecord, - failed: HealthCheck, - probe: Effect.Effect, + const stop = Effect.fn("Service.stop")(function* ( + generation: number, + stopOptions: { readonly operation: ServiceOperation; readonly discard: boolean }, ) { - const next = yield* Deferred.make>(); - const check = yield* Ref.modify(record.health, (latest): [HealthCheck, HealthCheck] => - latest === failed ? [next, next] : [latest, latest], + yield* Effect.annotateCurrentSpan({ member_id: id, generation }); + // Fencing under the execution lock waits out an acquisition already in progress, so its + // session is visible below, and makes any later acquisition for this generation a no-op. + // Only then is the attempt interrupted, while it prepares or checks readiness. + yield* execution.withPermit( + Ref.update(stoppedThrough, (value) => Math.max(value, generation)), + ); + const pending = yield* Ref.get(attempt); + if (pending !== undefined && pending.generation <= generation) + yield* Fiber.interrupt(pending.fiber); + const stopped = yield* execution + .withPermit( + Effect.gen(function* () { + const record = yield* Ref.get(current); + // A newer session belongs to a later generation's own stop. + if (record === undefined || record.generation > generation) return; + yield* update({ currentOperation: stopOptions.operation }); + yield* stopRecord(record, stopOptions.discard).pipe( + Effect.ensuring(update({ currentOperation: undefined })), + ); + }), + ) + .pipe(Effect.exit); + yield* report( + Exit.isSuccess(stopped) + ? LifecycleEvent.Exited({ id, generation, cause: undefined, requested: true }) + : LifecycleEvent.StopFailed({ id, generation, cause: Cause.squash(stopped.cause) }), ); - if (check === next) { - // stopNow fails the latest check after leaving "running", so a probe admitted here is always settled. - if (yield* isLive(record)) yield* runCheck(record, probe, next); - else yield* Deferred.succeed(next, Exit.fail(sessionStopped())); - } - return yield* Deferred.await(check); }); - const ready = Effect.fn("Service.ready")(function* () { - yield* Effect.annotateCurrentSpan({ member_id: options.id }); - const initial = yield* SubscriptionRef.get(observations); - if (!initial.registered) return yield* new ServiceDestroyed({ id: options.id }); - const expectedRevision = yield* Ref.get(revision); - const expectedLaunchId = initial.lifecycle === "starting" ? initial.launchId : undefined; - let record = yield* Ref.get(current); - if (record === undefined && initial.lifecycle === "starting") { - yield* run(Effect.void); - record = yield* Ref.get(current); - } - if (record === undefined) { - if ((yield* Ref.get(revision)) !== expectedRevision) - return yield* staleLaunch( - yield* Ref.get(launchCounter), - `Service ${options.id} changed before it was ready`, - ); - return yield* new ServiceNotRunning({ id: options.id }); - } - const launchId = record.launchId; - if ((yield* Ref.get(revision)) !== expectedRevision) - return yield* staleLaunch(launchId, `Service ${options.id} changed before it was ready`); - if (expectedLaunchId !== undefined && record.launchId !== expectedLaunchId) - return yield* staleLaunch(launchId, `Service ${options.id} changed before it was ready`); - if ((yield* SubscriptionRef.get(observations)).lifecycle === "stopping") { - return yield* diedBeforeReady(launchId); - } - const check = yield* Ref.get(record.health); - const settledOnArrival = yield* Deferred.isDone(check); - const healthResult = yield* Deferred.await(check); - if (!(yield* isLive(record))) return yield* diedBeforeReady(launchId); + const reprobe = Effect.fn("Service.reprobe")(function* (generation: number) { + yield* Effect.annotateCurrentSpan({ member_id: id, generation }); + const record = yield* Ref.get(current); + if (record?.generation !== generation) return; const probe = record.runtime.probe; - if (Exit.isSuccess(healthResult) || !settledOnArrival || probe === undefined) - return yield* healthResult; - const reprobed = yield* reprobe(record, check, probe); - if (!(yield* isLive(record))) return yield* diedBeforeReady(launchId); - return yield* reprobed; + if (probe === undefined) { + const error = + (yield* SubscriptionRef.get(observations)).error ?? + new ServiceError({ operation: "health", message: `Service ${id} is not ready` }); + yield* report(LifecycleEvent.ReadinessLost({ id, generation, cause: error })); + return; + } + const result = yield* check(record, probe); + if (result !== undefined) yield* settleCheck(record, result, true); }); - const storage = Effect.fn("Service.storage")(function* ( - operation: Effect.Effect, - ) { - yield* Effect.annotateCurrentSpan({ member_id: options.id }); - return yield* run( - Effect.gen(function* () { - const observation = yield* SubscriptionRef.get(observations); - if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); - if (observation.lifecycle !== "stopped" || observation.wakeEnabled) { - return yield* new ServiceNotStopped({ - id: options.id, - lifecycle: observation.lifecycle, - message: `Service ${options.id} must be stopped with wake disabled before modifying data`, - }); - } - yield* coordinate("storage", setOperation("storage")); - return yield* operation.pipe(Effect.ensuring(setOperation(undefined))); - }), - ); - }); + const storage = (operation: Effect.Effect) => + execution + .withPermit( + update({ currentOperation: "storage" }).pipe( + Effect.andThen(operation), + Effect.ensuring(update({ currentOperation: undefined })), + ), + ) + .pipe(Effect.withSpan("Service.storage", { attributes: { member_id: id } })); - const destroy = Effect.fn("Service.destroy")(function* () { - yield* Effect.annotateCurrentSpan({ member_id: options.id }); - yield* run( + const removeData = Effect.fn("Service.removeData")(function* () { + yield* Effect.annotateCurrentSpan({ member_id: id }); + yield* execution.withPermit( Effect.gen(function* () { - const observation = yield* SubscriptionRef.get(observations); - if (!observation.registered) return; - yield* coordinate("destroy", invalidate()); - yield* setOperation("destroy"); + if (!(yield* SubscriptionRef.get(observations)).registered) return; + yield* update({ currentOperation: "destroy" }); yield* Effect.gen(function* () { - yield* stopNow(undefined, false, true); - { - const dataScope = yield* Scope.fork(owner, "parallel"); - yield* definition - .removeData(contextFor(options.id, yield* Ref.get(config), dataScope)) - .pipe( - Effect.tapError((error) => update({ error })), - Effect.ensuring(Scope.close(dataScope, Exit.void)), - ); - } - yield* update({ registered: false, lifecycle: "stopped" }); - }).pipe(Effect.ensuring(setOperation(undefined))); + const leftover = yield* Ref.get(current); + if (leftover !== undefined) yield* stopRecord(leftover, true); + const dataScope = yield* Scope.fork(owner, "parallel"); + yield* definition + .removeData({ id, config: yield* Ref.get(config), scope: dataScope }) + .pipe( + Effect.tapError((error) => update({ error })), + Effect.ensuring(Scope.close(dataScope, Exit.void)), + ); + yield* update({ registered: false }); + }).pipe(Effect.ensuring(update({ currentOperation: undefined }))); }), ); }); return { - id: options.id, + id, observation: SubscriptionRef.changes(observations), get: SubscriptionRef.get(observations), - start, - startAt, - arm, - armAt, - sleep: sleep(), - stop: stop(), - restart, - ready: ready(), + prepare: (candidate) => definition.prepare?.(candidate) ?? Effect.void, + launch, + stop, + reprobe, storage, - destroy: destroy(), + removeData: removeData(), }; }); diff --git a/packages/stack/src/composition/Supabase.ts b/packages/stack/src/composition/Supabase.ts index 7c2c4431fa..73fa2aee22 100644 --- a/packages/stack/src/composition/Supabase.ts +++ b/packages/stack/src/composition/Supabase.ts @@ -624,9 +624,7 @@ export const makeSupabaseComposition = Effect.fn("Supabase.compose")( creation.service !== "database" && endpointNames(creation).length > 0; return lazy - ? creation.service === "functions" - ? { id, activation: "lazy" as const } - : { id, activation: "lazy" as const, idleMillis: idleMillisFor(creation.service) } + ? { id, activation: "lazy" as const, idleMillis: idleMillisFor(creation.service) } : { id, activation: "eager" as const }; }); yield* operations.configure({ diff --git a/packages/stack/src/composition/Supabase.unit.test.ts b/packages/stack/src/composition/Supabase.unit.test.ts index 8d0e259c90..2e6975f0fe 100644 --- a/packages/stack/src/composition/Supabase.unit.test.ts +++ b/packages/stack/src/composition/Supabase.unit.test.ts @@ -31,22 +31,21 @@ const makeInstance = ( }), removeData: () => Effect.void, }, - { id, config: {}, coordinate: orchestrator.admissionFor(id) }, + { id, config: {}, report: orchestrator.report }, ); - const instance: Orchestrator.RegisteredInstance = { + yield* orchestrator.register({ id, service: id, core, - startAt: (revision, inputs, wake, guard) => core.startAt(revision, inputs, wake, guard), - restart: (revision, inputs, config, guard) => core.restart(inputs, revision, guard), + launch: (generation, inputs) => core.launch(generation, inputs), + prepare: () => Effect.void, bind: Effect.void, close: Effect.void, hasEndpoint: options.hasEndpoint ?? true, inputs: options.inputs ?? [], outputs: options.outputs ?? {}, - }; - yield* orchestrator.register(instance); - return instance; + }); + return { status: orchestrator.status(id), observation: orchestrator.changes(id) }; }); /** Merges optional config bindings onto a creation; the config union can't express this generically. */ @@ -55,8 +54,8 @@ const withValues = ( values: Record, ): C => ({ ...creation, config: { ...creation.config, ...values } }) as C; -const stopped = (instance: Orchestrator.RegisteredInstance) => - instance.core.observation.pipe( +const stopped = (instance: Effect.Success>) => + instance.observation.pipe( Stream.filter((state) => state.lifecycle === "stopped" && state.currentOperation === undefined), Stream.take(1), Stream.runDrain, @@ -141,9 +140,9 @@ it.live( const requestScope = yield* Scope.make(); yield* orchestrator.acquire(studioId).pipe(Scope.provide(requestScope)); yield* TestClock.adjust("1 second"); - expect((yield* database.core.get).lifecycle).toBe("running"); - expect((yield* pgmeta.core.get).lifecycle).toBe("running"); - expect((yield* studio.core.get).lifecycle).toBe("running"); + expect((yield* database.status).lifecycle).toBe("running"); + expect((yield* pgmeta.status).lifecycle).toBe("running"); + expect((yield* studio.status).lifecycle).toBe("running"); // The request finished; studio itself now idles on its own 5-minute timer. yield* Scope.close(requestScope, Exit.void); @@ -151,18 +150,22 @@ it.live( // Well past pgmeta's own 60s idle mark but within studio's 5-minute window: studio // isn't idle yet, so the dependent-blocks-sleep rule keeps pgmeta running too. yield* TestClock.adjust("90 seconds"); - expect((yield* studio.core.get).lifecycle).toBe("running"); - expect((yield* pgmeta.core.get).lifecycle).toBe("running"); + expect((yield* studio.status).lifecycle).toBe("running"); + expect((yield* pgmeta.status).lifecycle).toBe("running"); // One second before studio's 5-minute idle deadline, measured from the request. yield* TestClock.adjust("208 seconds"); - expect((yield* studio.core.get).lifecycle).toBe("running"); - expect((yield* pgmeta.core.get).lifecycle).toBe("running"); + expect((yield* studio.status).lifecycle).toBe("running"); + expect((yield* pgmeta.status).lifecycle).toBe("running"); const studioStopped = yield* stopped(studio).pipe(Effect.forkChild); const pgmetaStopped = yield* stopped(pgmeta).pipe(Effect.forkChild); yield* TestClock.adjust("2 seconds"); yield* Fiber.join(studioStopped); + expect((yield* pgmeta.status).lifecycle).toBe("running"); + + // Once studio's stop is confirmed, pgmeta idles on its own timer. + yield* TestClock.adjust("60 seconds"); yield* Fiber.join(pgmetaStopped); }), ).pipe(Effect.provide(TestClock.layer())), diff --git a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts index 314151097c..5686a98f1a 100644 --- a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts +++ b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts @@ -2,7 +2,7 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Layer, Redacted } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; -import { makeService } from "../Service.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; @@ -58,7 +58,7 @@ describe("service catalog", () => { }, dockerOptions(root), ); - const database = yield* makeService(databaseRecipe.definition, { + const database = yield* makeStandaloneService(databaseRecipe.definition, { id: "database", config: databaseRecipe.creation, }); @@ -74,7 +74,7 @@ describe("service catalog", () => { }, dockerOptions(root), ); - const analytics = yield* makeService(analyticsRecipe.definition, { + const analytics = yield* makeStandaloneService(analyticsRecipe.definition, { id: "analytics", config: analyticsRecipe.creation, }); @@ -105,7 +105,7 @@ describe("service catalog", () => { "sources:\n dummy:\n type: demo_logs\n format: syslog\n interval: 60\n" + "sinks:\n print:\n type: console\n inputs: [dummy]\n encoding:\n codec: json\n", ); - const vector = yield* makeService(vectorRecipe.definition, { + const vector = yield* makeStandaloneService(vectorRecipe.definition, { id: "vector", config: vectorRecipe.creation, }); @@ -123,7 +123,7 @@ describe("service catalog", () => { { service: "imgproxy", config: { filePath: imageRoot } }, dockerOptions(root), ); - const imgproxy = yield* makeService(imgproxyRecipe.definition, { + const imgproxy = yield* makeStandaloneService(imgproxyRecipe.definition, { id: "imgproxy", config: imgproxyRecipe.creation, }); diff --git a/packages/stack/src/services/AuthStorage.integration.test.ts b/packages/stack/src/services/AuthStorage.integration.test.ts index 7becca6f13..981739094c 100644 --- a/packages/stack/src/services/AuthStorage.integration.test.ts +++ b/packages/stack/src/services/AuthStorage.integration.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Layer, Redacted, Schema } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { SignJWT } from "jose"; -import { makeService } from "../Service.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; @@ -58,7 +58,7 @@ describe("service catalog", () => { }, dockerOptions(root), ); - const database = yield* makeService(databaseRecipe.definition, { + const database = yield* makeStandaloneService(databaseRecipe.definition, { id: "database", config: databaseRecipe.creation, }); @@ -73,7 +73,7 @@ describe("service catalog", () => { }, dockerOptions(root), ); - const auth = yield* makeService(authRecipe.definition, { + const auth = yield* makeStandaloneService(authRecipe.definition, { id: "auth", config: authRecipe.creation, }); @@ -114,7 +114,7 @@ describe("service catalog", () => { { service: "imgproxy", config: { filePath: storageRoot } }, dockerOptions(root), ); - const imgproxy = yield* makeService(imgproxyRecipe.definition, { + const imgproxy = yield* makeStandaloneService(imgproxyRecipe.definition, { id: "imgproxy", config: imgproxyRecipe.creation, }); @@ -133,7 +133,7 @@ describe("service catalog", () => { }, dockerOptions(root), ); - const storage = yield* makeService(storageRecipe.definition, { + const storage = yield* makeStandaloneService(storageRecipe.definition, { id: "storage", config: storageRecipe.creation, }); diff --git a/packages/stack/src/services/Database.integration.test.ts b/packages/stack/src/services/Database.integration.test.ts index 60c42e3271..c51321da1d 100644 --- a/packages/stack/src/services/Database.integration.test.ts +++ b/packages/stack/src/services/Database.integration.test.ts @@ -15,7 +15,7 @@ import { } from "effect"; import { tmpdir } from "node:os"; import { DEFAULT_POSTGRES_ROOT_KEY } from "../Defaults.ts"; -import { makeService } from "../Service.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeDatabase, type BackendEndpoint, type DatabaseConfig } from "./Database.ts"; import { makeDockerDatabaseRoot, runDocker } from "../../tests/docker-fixture.ts"; import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; @@ -88,7 +88,7 @@ describe("database component", { timeout: 180_000 }, () => { jwtSecret: config.jwtSecret, jwtExpiry: config.jwtExpiry, }; - const service = yield* makeService(recipe.definition, { + const service = yield* makeStandaloneService(recipe.definition, { id: "database", config: defaults, }); @@ -141,7 +141,7 @@ describe("database component", { timeout: 180_000 }, () => { cacheRoot: artifactCacheRoot, runtime: target.runtime, }); - const secondService = yield* makeService(second.definition, { + const secondService = yield* makeStandaloneService(second.definition, { id: "database-second", config: { ...defaults, rootKey: Redacted.make("b".repeat(64)) }, }); @@ -177,7 +177,10 @@ describe("database component", { timeout: 180_000 }, () => { cacheRoot: artifactCacheRoot, runtime: "native", }); - const service = yield* makeService(database.definition, { id: "database:hba", config }); + const service = yield* makeStandaloneService(database.definition, { + id: "database:hba", + config, + }); yield* service.start; yield* service.ready; const endpoint = yield* database.endpoint; @@ -239,7 +242,7 @@ describe("database component", { timeout: 180_000 }, () => { runtime: "native", }); const loggedConfig: DatabaseConfig = { ...config, settings: { log_statement: "all" } }; - const service = yield* makeService(first.definition, { + const service = yield* makeStandaloneService(first.definition, { id: "database:first", config: loggedConfig, }); @@ -316,7 +319,7 @@ describe("database component", { timeout: 180_000 }, () => { cacheRoot, runtime: "native", }); - const reopenedService = yield* makeService(reopened.definition, { + const reopenedService = yield* makeStandaloneService(reopened.definition, { id: "database:first-reopened", config, }); @@ -344,7 +347,7 @@ describe("database component", { timeout: 180_000 }, () => { cacheRoot, runtime: "native", }); - const secondService = yield* makeService(second.definition, { + const secondService = yield* makeStandaloneService(second.definition, { id: "database:second", config, }); @@ -401,7 +404,7 @@ describe("database component", { timeout: 180_000 }, () => { runtime: "docker", engineTarget: dockerEngineTarget, }); - const service = yield* makeService(database.definition, { + const service = yield* makeStandaloneService(database.definition, { id: "database:container", config: databaseConfig, }); @@ -460,7 +463,7 @@ describe("database component", { timeout: 180_000 }, () => { engineTarget: dockerEngineTarget, }); const replacementPassword = Redacted.make("reopened-target-password"); - const reopenedService = yield* makeService(reopened.definition, { + const reopenedService = yield* makeStandaloneService(reopened.definition, { id: "database:reopened", config: { ...databaseConfig, databasePassword: replacementPassword }, }); @@ -498,7 +501,7 @@ describe("database component", { timeout: 180_000 }, () => { runtime: "docker", engineTarget: dockerEngineTarget, }); - const service = yield* makeService(database.definition, { + const service = yield* makeStandaloneService(database.definition, { id: "database:group", config, }); @@ -543,7 +546,7 @@ describe("database component", { timeout: 180_000 }, () => { runtime: "docker", engineTarget: dockerEngineTarget, }); - const service = yield* makeService(database.definition, { + const service = yield* makeStandaloneService(database.definition, { id: "database:shutdown", config: { ...config, stopGraceSeconds: 30 }, }); diff --git a/packages/stack/src/services/Functions.integration.test.ts b/packages/stack/src/services/Functions.integration.test.ts index a3a6be472e..080624a3a3 100644 --- a/packages/stack/src/services/Functions.integration.test.ts +++ b/packages/stack/src/services/Functions.integration.test.ts @@ -18,7 +18,7 @@ import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import type { FunctionsBootstrapOwner } from "../functions/FunctionsBootstrap.ts"; import { ContainerError, type ContainerRuntime } from "../runtime/Container.ts"; -import { makeService } from "../Service.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import * as Functions from "./Functions.ts"; import { makeProcessRecipe } from "./ProcessRecipe.ts"; @@ -114,7 +114,7 @@ describe("service catalog", () => { instanceId, }, ); - const instance = yield* makeService(recipe.definition, { + const instance = yield* makeStandaloneService(recipe.definition, { id: instanceId, config: recipe.creation, }); @@ -283,7 +283,7 @@ describe("service catalog", () => { ), Effect.forkScoped({ startImmediately: true }), ); - const instance = yield* makeService(recipe.definition, { + const instance = yield* makeStandaloneService(recipe.definition, { id: "ancestor", config: recipe.creation, }); @@ -357,7 +357,7 @@ describe("service catalog", () => { ), Effect.forkScoped({ startImmediately: true }), ); - const instance = yield* makeService(recipe.definition, { + const instance = yield* makeStandaloneService(recipe.definition, { id: "deno-config", config: recipe.creation, }); @@ -441,7 +441,7 @@ describe("service catalog", () => { ), Effect.forkScoped({ startImmediately: true }), ); - const instance = yield* makeService(recipe.definition, { + const instance = yield* makeStandaloneService(recipe.definition, { id: "plain-deno-config", config: recipe.creation, }); @@ -552,7 +552,7 @@ for (const runtime of ["native", "docker"] as const) { ), Effect.forkScoped({ startImmediately: true }), ); - const instance = yield* makeService(recipe.definition, { + const instance = yield* makeStandaloneService(recipe.definition, { id: "configured", config: recipe.creation, }); diff --git a/packages/stack/src/services/Mail.integration.test.ts b/packages/stack/src/services/Mail.integration.test.ts index c19305e8dc..75a456cdcd 100644 --- a/packages/stack/src/services/Mail.integration.test.ts +++ b/packages/stack/src/services/Mail.integration.test.ts @@ -2,7 +2,7 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Effect, Exit, FileSystem, Layer, Path } from "effect"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; -import { makeService } from "../Service.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { noContainerClaims, @@ -75,7 +75,7 @@ describe("service catalog", () => { { service: "mail", config: {} }, dockerOptions(root), ); - const instance = yield* makeService(recipe.definition, { + const instance = yield* makeStandaloneService(recipe.definition, { id: "mail", config: recipe.creation, }); @@ -103,11 +103,11 @@ describe("service catalog", () => { makeServiceRecipe({ service: "mail", config: {} }, { ...options(root), instanceId }); const first = yield* recipeFor("mail-a"); const second = yield* recipeFor("mail-b"); - const firstInstance = yield* makeService(first.definition, { + const firstInstance = yield* makeStandaloneService(first.definition, { id: "mail-a", config: first.creation, }); - const secondInstance = yield* makeService(second.definition, { + const secondInstance = yield* makeStandaloneService(second.definition, { id: "mail-b", config: second.creation, }); @@ -132,7 +132,7 @@ describe("service catalog", () => { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-mail-persistence-" }); const recipe = yield* makeServiceRecipe({ service: "mail", config: {} }, options(root)); - const instance = yield* makeService(recipe.definition, { + const instance = yield* makeStandaloneService(recipe.definition, { id: "mail", config: recipe.creation, }); @@ -166,7 +166,7 @@ describe("service catalog", () => { { service: "mail", config: {} }, dockerOptions(root), ); - const instance = yield* makeService(recipe.definition, { + const instance = yield* makeStandaloneService(recipe.definition, { id: "mail", config: recipe.creation, }); @@ -196,7 +196,7 @@ describe("service catalog", () => { const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-mail-destroy-" }); const recipe = yield* makeServiceRecipe({ service: "mail", config: {} }, options(root)); - const instance = yield* makeService(recipe.definition, { + const instance = yield* makeStandaloneService(recipe.definition, { id: "mail", config: recipe.creation, }); @@ -220,7 +220,7 @@ describe("service catalog", () => { { service: "mail", config: {} }, { ...options(root), instanceId: "../escaped" }, ); - const instance = yield* makeService(recipe.definition, { + const instance = yield* makeStandaloneService(recipe.definition, { id: "mail", config: recipe.creation, }); diff --git a/packages/stack/src/services/Pooler.integration.test.ts b/packages/stack/src/services/Pooler.integration.test.ts index e3113642a0..b6b5a169dc 100644 --- a/packages/stack/src/services/Pooler.integration.test.ts +++ b/packages/stack/src/services/Pooler.integration.test.ts @@ -2,7 +2,7 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Context, Effect, Layer, Redacted } from "effect"; import { PgClient } from "@effect/sql-pg"; -import { makeService } from "../Service.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { ProxyError } from "../Proxy.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerTcpRelay } from "../../tests/docker-relay.ts"; @@ -52,7 +52,7 @@ describe("service catalog", () => { }, dockerOptions(root), ); - const database = yield* makeService(databaseRecipe.definition, { + const database = yield* makeStandaloneService(databaseRecipe.definition, { id: "database", config: databaseRecipe.creation, }); @@ -81,7 +81,7 @@ describe("service catalog", () => { }, runtime === "native" ? options(root) : dockerOptions(root), ); - const pooler = yield* makeService(poolerRecipe.definition, { + const pooler = yield* makeStandaloneService(poolerRecipe.definition, { id: `pooler-${runtime}-${poolMode}`, config: poolerRecipe.creation, }); diff --git a/packages/stack/src/services/ProcessRecipe.integration.test.ts b/packages/stack/src/services/ProcessRecipe.integration.test.ts index 29724fbead..3b2f9bc8e5 100644 --- a/packages/stack/src/services/ProcessRecipe.integration.test.ts +++ b/packages/stack/src/services/ProcessRecipe.integration.test.ts @@ -38,7 +38,8 @@ import { type ContainerProcess, type ContainerRuntime, } from "../runtime/Container.ts"; -import { makeService, ServiceError } from "../Service.ts"; +import { ServiceError } from "../Service.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeProcessRecipe, type ProcessDependencies, @@ -201,7 +202,7 @@ describe("ProcessRecipe launch cleanup", () => { isPubliclyReserved: noPublicPortReservations, } satisfies ProcessDependencies; const recipe = yield* makeProcessRecipe(creation, options, dependencies, spec); - const service = yield* makeService(recipe.definition, { + const service = yield* makeStandaloneService(recipe.definition, { id: `rest-process-recipe-${scenario}`, config: creation, }); @@ -290,7 +291,7 @@ describe("ProcessRecipe launch cleanup", () => { isPubliclyReserved: noPublicPortReservations, } satisfies ProcessDependencies; const recipe = yield* makeProcessRecipe(creation, nativeOptions, dependencies, nativeSpec); - const service = yield* makeService(recipe.definition, { + const service = yield* makeStandaloneService(recipe.definition, { id: "rest-process-recipe-native", config: creation, }); @@ -402,7 +403,7 @@ describe("ProcessRecipe launch cleanup", () => { isPubliclyReserved: noPublicPortReservations, } satisfies ProcessDependencies; const recipe = yield* makeProcessRecipe(creation, nativeOptions, dependencies, nativeSpec); - const service = yield* makeService(recipe.definition, { + const service = yield* makeStandaloneService(recipe.definition, { id: "rest-process-recipe-port-order", config: creation, }); @@ -485,7 +486,7 @@ const realtimeService = Effect.fn(function* (container: ContainerRuntime) { }, Realtime.makeSpec(), ); - return yield* makeService(recipe.definition, { id: "realtime", config: creation }); + return yield* makeStandaloneService(recipe.definition, { id: "realtime", config: creation }); }); const platform = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); @@ -765,7 +766,10 @@ describe("process recipe startup", () => { `server.listen(Number(process.env.PORT), "127.0.0.1");\n`, collidingSpawner, ); - const service = yield* makeService(recipe.definition, { id: "rest", config: creation }); + const service = yield* makeStandaloneService(recipe.definition, { + id: "rest", + config: creation, + }); yield* service.start; yield* service.ready; @@ -819,7 +823,10 @@ describe("process recipe startup", () => { {}, (line) => line.includes("listener bound"), ); - const service = yield* makeService(recipe.definition, { id: "rest", config: creation }); + const service = yield* makeStandaloneService(recipe.definition, { + id: "rest", + config: creation, + }); yield* service.start; const failure = yield* Effect.flip(service.ready); @@ -864,7 +871,10 @@ describe("process recipe startup", () => { spawner, { PID_FILE: pidFile }, ); - const service = yield* makeService(recipe.definition, { id: "rest", config: creation }); + const service = yield* makeStandaloneService(recipe.definition, { + id: "rest", + config: creation, + }); yield* service.start; const failure = yield* Effect.flip(service.ready); @@ -1055,7 +1065,10 @@ describe("process recipe startup", () => { }, Pooler.makeSpec(), ); - const service = yield* makeService(recipe.definition, { id: "pooler", config: creation }); + const service = yield* makeStandaloneService(recipe.definition, { + id: "pooler", + config: creation, + }); const subscribed = yield* Deferred.make(); const exitObserved = yield* Deferred.make(); yield* service.observation.pipe( diff --git a/packages/stack/src/services/ProcessRecipe.ts b/packages/stack/src/services/ProcessRecipe.ts index 9140d5f3e4..c75eccfc31 100644 --- a/packages/stack/src/services/ProcessRecipe.ts +++ b/packages/stack/src/services/ProcessRecipe.ts @@ -221,6 +221,11 @@ const emptyNativePortSet: ReadonlySet = new Set(); const startupTimeoutSeconds = 60; const nativeLaunchAttempts = 3; const probeTimeout = Duration.seconds(10); +/** + * Bounds one readiness HTTP call: a request that connects but never answers is retried instead + * of spending the whole check on that one attempt. + */ +const readinessAttemptTimeout = Duration.seconds(3); const outputDrainGrace = Duration.seconds(2); const startupOutputTailLines = 20; const startupOutputLineChars = 1_000; @@ -386,6 +391,7 @@ const readiness = Effect.fn("ProcessRecipe.readiness")(function* ( new ServiceError({ operation: "health", message: `HTTP ${response.status}` }), ), ), + Effect.timeout(readinessAttemptTimeout), ); return yield* withAttemptCount(attempt, (counted) => counted.pipe( diff --git a/packages/stack/src/services/Realtime.integration.test.ts b/packages/stack/src/services/Realtime.integration.test.ts index b60f506e77..664e9373dd 100644 --- a/packages/stack/src/services/Realtime.integration.test.ts +++ b/packages/stack/src/services/Realtime.integration.test.ts @@ -2,7 +2,7 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Effect, Layer, Redacted } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; -import { makeService } from "../Service.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; @@ -51,7 +51,7 @@ describe("service catalog", () => { }, dockerOptions(root), ); - const database = yield* makeService(databaseRecipe.definition, { + const database = yield* makeStandaloneService(databaseRecipe.definition, { id: "database", config: databaseRecipe.creation, }); @@ -64,7 +64,7 @@ describe("service catalog", () => { { service: "realtime", config: { databaseUrl, jwtSecret: secret } }, dockerOptions(root), ); - const realtime = yield* makeService(realtimeRecipe.definition, { + const realtime = yield* makeStandaloneService(realtimeRecipe.definition, { id: "realtime", config: realtimeRecipe.creation, }); @@ -82,7 +82,7 @@ describe("service catalog", () => { { service: "pgmeta", config: { databaseUrl } }, dockerOptions(root), ); - const pgmeta = yield* makeService(pgmetaRecipe.definition, { + const pgmeta = yield* makeStandaloneService(pgmetaRecipe.definition, { id: "pgmeta", config: pgmetaRecipe.creation, }); @@ -110,7 +110,7 @@ describe("service catalog", () => { }, dockerOptions(root), ); - const studio = yield* makeService(studioRecipe.definition, { + const studio = yield* makeStandaloneService(studioRecipe.definition, { id: "studio", config: studioRecipe.creation, }); diff --git a/packages/stack/src/services/Rest.integration.test.ts b/packages/stack/src/services/Rest.integration.test.ts index a9d0e9d0ef..97e3746145 100644 --- a/packages/stack/src/services/Rest.integration.test.ts +++ b/packages/stack/src/services/Rest.integration.test.ts @@ -7,7 +7,7 @@ import { SignJWT } from "jose"; import { tmpdir } from "node:os"; import * as Network from "../Network.ts"; import * as StackNamespace from "../StackNamespace.ts"; -import { makeService } from "../Service.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { ProxyError } from "../Proxy.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; @@ -85,7 +85,7 @@ describe("service catalog", () => { }, dockerOptions(root), ); - const database = yield* makeService(databaseRecipe.definition, { + const database = yield* makeStandaloneService(databaseRecipe.definition, { id: "database", config: databaseRecipe.creation, }); @@ -150,7 +150,7 @@ describe("service catalog", () => { }, dockerOptions(root), ); - const rest = yield* makeService(restRecipe.definition, { + const rest = yield* makeStandaloneService(restRecipe.definition, { id: "rest", config: restRecipe.creation, }); @@ -227,7 +227,7 @@ describe("service catalog", () => { }, { ...options(root), stackId, cacheRoot: `${tmpdir()}/supabase-stack-artifacts` }, ); - const database = yield* makeService(databaseRecipe.definition, { + const database = yield* makeStandaloneService(databaseRecipe.definition, { id: "database", config: databaseRecipe.creation, }); @@ -267,7 +267,7 @@ describe("service catalog", () => { }, { ...options(root), stackId, cacheRoot: `${tmpdir()}/supabase-stack-artifacts` }, ); - const rest = yield* makeService(restRecipe.definition, { + const rest = yield* makeStandaloneService(restRecipe.definition, { id: "rest", config: restRecipe.creation, }); diff --git a/packages/stack/src/services/Studio.integration.test.ts b/packages/stack/src/services/Studio.integration.test.ts index 17974e0c42..d114bd9ec9 100644 --- a/packages/stack/src/services/Studio.integration.test.ts +++ b/packages/stack/src/services/Studio.integration.test.ts @@ -1,7 +1,7 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Layer } from "effect"; -import { makeService } from "../Service.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { noContainerClaims, @@ -36,7 +36,7 @@ describe("studio recipe", () => { { service: "studio", config: { functionsRoot: siblingInstance } }, options(root), ); - const studio = yield* makeService(recipe.definition, { + const studio = yield* makeStandaloneService(recipe.definition, { id: "studio", config: recipe.creation, }); diff --git a/packages/stack/src/services/Vector.integration.test.ts b/packages/stack/src/services/Vector.integration.test.ts index d38026d953..44624817f2 100644 --- a/packages/stack/src/services/Vector.integration.test.ts +++ b/packages/stack/src/services/Vector.integration.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "@effect/vitest"; import { tmpdir } from "node:os"; import { Effect, FileSystem, Layer } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; -import { makeService } from "../Service.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { noContainerClaims, @@ -44,7 +44,7 @@ describe("vector recipe", () => { }, options(root, runtime), ); - const vector = yield* makeService(recipe.definition, { + const vector = yield* makeStandaloneService(recipe.definition, { id: "vector", config: recipe.creation, }); @@ -89,7 +89,7 @@ describe("vector recipe", () => { }, options(root, "docker"), ); - const vector = yield* makeService(recipe.definition, { + const vector = yield* makeStandaloneService(recipe.definition, { id: "vector", config: recipe.creation, }); @@ -119,7 +119,7 @@ describe("vector recipe", () => { }, options(root, "docker"), ); - const vector = yield* makeService(recipe.definition, { + const vector = yield* makeStandaloneService(recipe.definition, { id: "vector", config: recipe.creation, }); @@ -173,7 +173,7 @@ describe("vector recipe", () => { }, options(root, "docker"), ); - const vector = yield* makeService(recipe.definition, { + const vector = yield* makeStandaloneService(recipe.definition, { id: "vector", config: recipe.creation, }); diff --git a/packages/stack/src/testing.integration.test.ts b/packages/stack/src/testing.integration.test.ts index a193eb6a99..f35d4e1311 100644 --- a/packages/stack/src/testing.integration.test.ts +++ b/packages/stack/src/testing.integration.test.ts @@ -169,6 +169,56 @@ it.live( { timeout: 120_000 }, ); +it.live( + "destroys a stack after the snapshot running at the time finishes, removing its data", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateRoot = yield* fs.makeTempDirectoryScoped({ prefix: "stack-testing-snapshot-" }); + const test = yield* makeTestStack({ runtime: "native", stateRoot }); + const database = test.services.database; + const { databaseUrl } = yield* database.credentials(); + if (databaseUrl === undefined) return yield* Effect.die("Database URL missing"); + // Enough data that the snapshot is still copying when the destroy arrives. + yield* sql( + test.stack, + databaseUrl, + "CREATE TABLE filler AS SELECT g, repeat('x', 200) AS value FROM generate_series(1, 200000) g", + ); + yield* database.stop; + const subscribed = yield* Deferred.make(); + const storing = yield* Deferred.make(); + yield* database.followStatus.pipe( + Stream.runForEach((status) => + Deferred.succeed(subscribed, undefined).pipe( + Effect.andThen( + status.currentOperation === "storage" + ? Deferred.succeed(storing, undefined) + : Effect.void, + ), + ), + ), + Effect.forkScoped, + ); + yield* Deferred.await(subscribed); + + const snapshot = yield* database + .saveSnapshot("before-destroy", { scope: "instance" }) + .pipe(Effect.forkScoped); + yield* Deferred.await(storing); + yield* test.stack.destroy; + + expect(Exit.isSuccess(yield* Fiber.await(snapshot))).toBe(true); + expect(yield* discover({ stateRoot })).toEqual([]); + expect(yield* fs.exists(path.join(stateRoot, test.stack.id))).toBe(false); + }).pipe( + Effect.scoped, + Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp)), + ), + { timeout: 120_000 }, +); + it.live( "names the failure, service states and owner log, then removes the stack when test startup fails", () => diff --git a/packages/stack/src/whole-stack.docker.idle-parallel.e2e.test.ts b/packages/stack/src/whole-stack.docker.idle-parallel.e2e.test.ts index a18f68ff23..e3d6e380e3 100644 --- a/packages/stack/src/whole-stack.docker.idle-parallel.e2e.test.ts +++ b/packages/stack/src/whole-stack.docker.idle-parallel.e2e.test.ts @@ -1,9 +1,15 @@ import { it } from "@effect/vitest"; import { parallel } from "../tests/whole-stack/scenarios.ts"; -import { idleWake } from "../tests/whole-stack/idle.ts"; +import { functionsIdleWake, idleWake, restartIdleWake } from "../tests/whole-stack/idle.ts"; import { run } from "../tests/whole-stack/fixture.ts"; it.live("Docker: idle and wake", () => run(idleWake("docker")), { timeout: 10 * 60_000 }); +it.live("Docker: restart preserves wake and idles again", () => run(restartIdleWake("docker")), { + timeout: 10 * 60_000, +}); +it.live("Docker: functions idles and wakes", () => run(functionsIdleWake("docker")), { + timeout: 10 * 60_000, +}); it.live("Docker: parallel stack isolation", () => run(parallel("docker")), { timeout: 30 * 60_000, }); diff --git a/packages/stack/src/whole-stack.native.idle-parallel.e2e.test.ts b/packages/stack/src/whole-stack.native.idle-parallel.e2e.test.ts index 16fdcce8be..0993cf6e2e 100644 --- a/packages/stack/src/whole-stack.native.idle-parallel.e2e.test.ts +++ b/packages/stack/src/whole-stack.native.idle-parallel.e2e.test.ts @@ -1,9 +1,15 @@ import { it } from "@effect/vitest"; import { parallel } from "../tests/whole-stack/scenarios.ts"; -import { idleWake } from "../tests/whole-stack/idle.ts"; +import { functionsIdleWake, idleWake, restartIdleWake } from "../tests/whole-stack/idle.ts"; import { run } from "../tests/whole-stack/fixture.ts"; it.live("native: idle and wake", () => run(idleWake("native")), { timeout: 10 * 60_000 }); +it.live("native: restart preserves wake and idles again", () => run(restartIdleWake("native")), { + timeout: 10 * 60_000, +}); +it.live("native: functions idles and wakes", () => run(functionsIdleWake("native")), { + timeout: 10 * 60_000, +}); it.live("native: parallel stack isolation", () => run(parallel("native")), { timeout: 30 * 60_000, }); diff --git a/packages/stack/tests/docker-fixture.integration.test.ts b/packages/stack/tests/docker-fixture.integration.test.ts index 8c8e7c4943..3117a36eca 100644 --- a/packages/stack/tests/docker-fixture.integration.test.ts +++ b/packages/stack/tests/docker-fixture.integration.test.ts @@ -3,7 +3,7 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { Context, Effect, FileSystem, Layer, Redacted, Schema, Stream } from "effect"; -import { makeService } from "../src/Service.ts"; +import { makeStandaloneService } from "./standalone-service.ts"; import { makeDatabase, type BackendEndpoint } from "../src/services/Database.ts"; import { makeDockerDatabaseRoot } from "./docker-fixture.ts"; import { noContainerClaims, noDirectoryClaims } from "./claims.ts"; @@ -81,7 +81,7 @@ describe("Docker database fixture isolation", { timeout: 180_000 }, () => { }); const [first, second] = yield* Effect.all([make(firstRoot), make(secondRoot)]); const [firstService, secondService] = yield* Effect.all([ - makeService(first.definition, { + makeStandaloneService(first.definition, { id: "docker-fixture-first", config: { version: "17", @@ -90,7 +90,7 @@ describe("Docker database fixture isolation", { timeout: 180_000 }, () => { jwtExpiry: 3600, }, }), - makeService(second.definition, { + makeStandaloneService(second.definition, { id: "docker-fixture-second", config: { version: "17", diff --git a/packages/stack/tests/standalone-service.ts b/packages/stack/tests/standalone-service.ts new file mode 100644 index 0000000000..131f39709b --- /dev/null +++ b/packages/stack/tests/standalone-service.ts @@ -0,0 +1,60 @@ +import { Effect, Ref } from "effect"; +import * as Orchestrator from "../src/Orchestrator.ts"; +import { makeService, type ServiceDefinition, type ServiceError } from "../src/Service.ts"; + +/** + * Runs one service definition under its own lifecycle authority, as a lazy member with no idle + * timeout: an explicit start launches it once, and a failed launch is not retried on its own. + */ +export const makeStandaloneService = Effect.fn("Test.makeStandaloneService")(function* ( + definition: ServiceDefinition, + options: { readonly id: string; readonly config: Config }, +) { + const { id } = options; + const orchestrator = yield* Orchestrator.make(); + const current = yield* Ref.make(options.config); + const candidate = yield* Ref.make(options.config); + const core = yield* makeService(definition, { + id, + config: options.config, + report: orchestrator.report, + }); + // The orchestrator only passes a restart candidate through; the typed value lives here. + const configFor = (restarted: unknown) => + restarted === undefined ? Ref.get(current) : Ref.get(candidate); + yield* orchestrator.register({ + id, + service: id, + core, + launch: (generation, _inputs, restarted) => + configFor(restarted).pipe( + Effect.tap((config) => Ref.set(current, config)), + Effect.flatMap((config) => core.launch(generation, config)), + ), + prepare: (_inputs, restarted) => configFor(restarted).pipe(Effect.flatMap(core.prepare)), + bind: Effect.void, + close: Effect.void, + hasEndpoint: true, + inputs: [], + outputs: {}, + }); + yield* orchestrator.configure({ members: [{ id, activation: "lazy" }], dependencies: [] }); + return { + id, + get: Effect.all([orchestrator.status(id), core.get]).pipe( + Effect.map(([status, execution]) => ({ ...status, config: execution.config })), + ), + observation: orchestrator.changes(id), + /** Execution facts that outlive the instance's registration, such as `registered`. */ + execution: core.get, + start: orchestrator.start(id), + ready: orchestrator.ready(id), + stop: orchestrator.stop(id), + restart: (config?: Config) => + config === undefined + ? orchestrator.restart(id) + : Ref.set(candidate, config).pipe(Effect.andThen(orchestrator.restart(id, config))), + destroy: orchestrator.destroy(id), + storage: (operation: Effect.Effect) => orchestrator.storage(id, operation), + }; +}); diff --git a/packages/stack/tests/whole-stack/idle.ts b/packages/stack/tests/whole-stack/idle.ts index 67e3f46289..cc7e56b68b 100644 --- a/packages/stack/tests/whole-stack/idle.ts +++ b/packages/stack/tests/whole-stack/idle.ts @@ -11,6 +11,7 @@ import { waitForLifecycle, wholeStack, type Runtime, + type WholeStack, } from "./fixture.ts"; import { subscribeRealtime } from "./websocket.ts"; @@ -107,13 +108,13 @@ export const idleWake = (runtime: Runtime) => { id: fixture.stack.id, value: "awake" }, { ...headers, "content-type": "application/json", prefer: "return=minimal" }, ); - expect(insert.status).toBe(201); + expect(insert.status, insert.body).toBe(201); yield* Fiber.join(firstChange); const rest = service(fixture, "rest"); yield* waitForLifecycle(rest, "stopped"); expect((yield* rest.status).wakeEnabled).toBe(true); const authHealth = yield* requestWithHeaders(`${authUrl}/health`, {}); - expect(authHealth.status).toBe(200); + expect(authHealth.status, authHealth.body).toBe(200); expect((yield* rest.status).lifecycle).toBe("stopped"); expect((yield* database.status).lifecycle).toBe("running"); expect((yield* service(fixture, "realtime").status).lifecycle).toBe("running"); @@ -125,7 +126,7 @@ export const idleWake = (runtime: Runtime) => `${restUrl}/idle_items?id=eq.${fixture.stack.id}`, headers, ); - expect(wake.status).toBe(200); + expect(wake.status, wake.body).toBe(200); expect((yield* rest.status).lifecycle).toBe("running"); const marker = `${fixture.stack.id}-change`; const changed = yield* jsonRequest( @@ -134,7 +135,7 @@ export const idleWake = (runtime: Runtime) => { id: marker, value: "wake" }, { ...headers, "content-type": "application/json", prefer: "return=minimal" }, ); - expect(changed.status).toBe(201); + expect(changed.status, changed.body).toBe(201); const change = yield* Fiber.join(nextChange); const changeRow = yield* Schema.decodeUnknownEffect( Schema.Struct({ record: Schema.Struct({ id: Schema.String, value: Schema.String }) }), @@ -168,7 +169,7 @@ export const idleWake = (runtime: Runtime) => `${restUrl}/idle_items?id=eq.${fixture.stack.id}`, headers, ); - expect(wake.status).toBe(200); + expect(wake.status, wake.body).toBe(200); const marker = `${fixture.stack.id}-reconnect`; const changed = yield* jsonRequest( "POST", @@ -176,7 +177,7 @@ export const idleWake = (runtime: Runtime) => { id: marker, value: "reconnect" }, { ...headers, "content-type": "application/json", prefer: "return=minimal" }, ); - expect(changed.status).toBe(201); + expect(changed.status, changed.body).toBe(201); const change = yield* Fiber.join(nextChange); const changeRow = yield* Schema.decodeUnknownEffect( Schema.Struct({ record: Schema.Struct({ id: Schema.String, value: Schema.String }) }), @@ -188,3 +189,104 @@ export const idleWake = (runtime: Runtime) => yield* waitForLifecycle(service(fixture, "pooler"), "stopped"); }), ); + +/** Only the given member carries a short idle timeout; the rest of the composition stays lazy but untimed. */ +const configureSoleTimedMember = (fixture: WholeStack, timedId: string) => + Effect.gen(function* () { + const database = service(fixture, "database"); + const configuration = yield* fixture.stack.composition.describe; + yield* fixture.stack.composition.configure({ + ...configuration, + members: configuration.members.map((member) => + member.id === database.id + ? { id: member.id, activation: "eager" as const } + : member.id === timedId + ? { id: member.id, activation: "lazy" as const, idleMillis } + : { id: member.id, activation: "lazy" as const }, + ), + }); + }); + +/** A restart through the public API must preserve activation intent: the member still idle-sleeps + * afterwards and wakes again on the next request (regression for Service.restart disarming wake). */ +export const restartIdleWake = (runtime: Runtime) => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* wholeStack(runtime); + const rest = service(fixture, "rest"); + yield* configureSoleTimedMember(fixture, rest.id); + yield* fixture.stack.composition.start; + + const token = yield* signServiceToken(fixture.secret); + const restUrl = (yield* rest.credentials()).url; + if (restUrl === undefined) return yield* Effect.die("Rest URL missing"); + const headers = { authorization: `Bearer ${token}`, apikey: token }; + + const warm = yield* requestWithHeaders(restUrl, headers); + expect(warm.status, warm.body).toBe(200); + expect((yield* rest.status).lifecycle).toBe("running"); + + yield* rest.restart(); + expect((yield* rest.status).wakeEnabled).toBe(true); + + yield* waitForLifecycle(rest, "stopped"); + expect((yield* rest.status).wakeEnabled).toBe(true); + + const rewarm = yield* requestWithHeaders(restUrl, headers); + expect(rewarm.status, rewarm.body).toBe(200); + expect((yield* rest.status).lifecycle).toBe("running"); + }), + ); + +/** Functions must idle-sleep and wake like every other lazy service (no exemption). */ +export const functionsIdleWake = (runtime: Runtime) => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* wholeStack(runtime); + const functionsInstance = service(fixture, "functions"); + // The default composition policy (not a test override) must give functions the same idle + // timeout as its peers instead of exempting it. + const defaultConfiguration = yield* fixture.stack.composition.describe; + const defaultMember = defaultConfiguration.members.find( + (member) => member.id === functionsInstance.id, + ); + expect(defaultMember?.activation).toBe("lazy"); + expect(defaultMember?.idleMillis).toBe(60_000); + yield* configureSoleTimedMember(fixture, functionsInstance.id); + yield* fixture.stack.composition.start; + yield* sql( + fixture, + "create table if not exists public.whole_stack_items (id text primary key, value text); grant select, insert on public.whole_stack_items to anon, authenticated, service_role;", + ); + + const token = yield* signServiceToken(fixture.secret); + const functionsUrl = (yield* functionsInstance.credentials()).url; + if (functionsUrl === undefined) return yield* Effect.die("Functions URL missing"); + const headers = { + authorization: `Bearer ${token}`, + apikey: token, + "content-type": "application/json", + }; + + const invoke = yield* jsonRequest( + "POST", + `${functionsUrl}/hello`, + { id: "functions-idle-probe" }, + headers, + ); + expect(invoke.status, invoke.body).toBe(200); + expect((yield* functionsInstance.status).lifecycle).toBe("running"); + + yield* waitForLifecycle(functionsInstance, "stopped"); + expect((yield* functionsInstance.status).wakeEnabled).toBe(true); + + const rewake = yield* jsonRequest( + "POST", + `${functionsUrl}/hello`, + { id: "functions-idle-probe" }, + headers, + ); + expect(rewake.status, rewake.body).toBe(200); + expect((yield* functionsInstance.status).lifecycle).toBe("running"); + }), + ); diff --git a/packages/stack/tests/whole-stack/scenarios.ts b/packages/stack/tests/whole-stack/scenarios.ts index 2b8faffe7c..19430852fe 100644 --- a/packages/stack/tests/whole-stack/scenarios.ts +++ b/packages/stack/tests/whole-stack/scenarios.ts @@ -60,9 +60,6 @@ const assertDefaultPolicy = (fixture: WholeStack) => if (name === "database") { expect(member.activation).toBe("eager"); expect(member.idleMillis).toBeUndefined(); - } else if (name === "functions") { - expect(member.activation).toBe("lazy"); - expect(member.idleMillis).toBeUndefined(); } else { expect(member.activation).toBe("lazy"); expect(member.idleMillis).toBe(name === "studio" ? 300_000 : 60_000); From e6ff9b85cbaa73e5f73132f13f5aafad53f63d6f Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 04:40:41 +0200 Subject: [PATCH 033/142] feat(stack): drain connections and exit on abandoned registration Shutdown drain (F5): owner stop and destroy now close accept on every stack listener before any service stops, let established connections (including HTTP keep-alive and pooled dependency connections) keep flowing until each closes or a 10s deadline passes, then cut the rest. Abandoned registration (F6): an owner now polls its own registration file every 30s (shortened only by a dedicated internal test fixture) and, on a confirmed ENOENT, drains, stops every workload, removes what it created through the same cleanup path as destroy, releases its port reservations and exits, without touching the now-gone registration. --- .oxlintrc.json | 3 +- .../stack/destroy/SIDE_EFFECTS.md | 10 +- .../experimental/stack/stop/SIDE_EFFECTS.md | 6 + packages/stack/ARCHITECTURE.md | 34 ++++- packages/stack/src/Network.ts | 76 ++++++++++- packages/stack/src/Orchestrator.ts | 3 + packages/stack/src/Owner.ts | 92 ++++++++++++- ...ost.container-shutdown.integration.test.ts | 76 +++++++++++ .../stack/src/StackHost.integration.test.ts | 128 ++++++++++++++++++ packages/stack/src/StackHost.ts | 98 +++++++++++++- .../stack/src/lifetime.integration.test.ts | 105 +++++++++++++- .../tests/short-registration-poll-fixture.ts | 100 ++++++++++++++ 12 files changed, 714 insertions(+), 17 deletions(-) create mode 100644 packages/stack/tests/short-registration-poll-fixture.ts diff --git a/.oxlintrc.json b/.oxlintrc.json index dee35ff5dc..f8e74bf71b 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -382,7 +382,8 @@ "packages/stack/src/functions/serve-main-resolver.integration.test.ts", "packages/stack/tests/effect-timeout.fixture.ts", "packages/stack/tests/host-process-fixture.ts", - "packages/stack/tests/failing-owner-fixture.ts" + "packages/stack/tests/failing-owner-fixture.ts", + "packages/stack/tests/short-registration-poll-fixture.ts" ], "rules": { "no-restricted-imports": [ diff --git a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md index c8d8170146..82a226fd8b 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md @@ -23,6 +23,10 @@ Cleanup failures remain errors; the command does not claim success on failure. A failed destroy may leave its owner running; retry destruction or use `stack stop` to shut down that owner. +As with `stack stop`, the owner drains live connections on every stack listener +(up to 10 seconds) before any service stops, so a request in flight when +destroy is issued can still complete. + When no owner is running and the engine reports that its daemon cannot be reached, destruction removes nothing: the registration, port claims and host data stay as they are, stderr warns that Docker resources were not removed, and @@ -47,8 +51,10 @@ override) only once cleanup is confirmed complete; a destroy that fails or skips engine cleanup keeps the stack's reservations, and its ports remain unavailable to other stacks until a later destroy succeeds. A reservation whose owning stack's state was deleted without going through destroy (for example a removed -or unmounted state root) is reclaimed lazily by whichever stack next needs that -port. +or unmounted state root) is usually released by that stack's own still-running +owner within its next registration check (30 seconds by default), which also +stops its workloads; the lazy reclaim by whichever stack next needs that port +remains the backstop when no owner is left alive to do so. ## Output, exit codes and telemetry diff --git a/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md index 9df0c80610..1bfd6e22e9 100644 --- a/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md @@ -16,6 +16,12 @@ command reports `No owner is reachable; workload state is unavailable.` It neither starts an owner nor claims workloads have stopped. If an owner disappears after that preflight, the shutdown error remains a failure. +Before any service stops, the owner stops accepting new connections on every +stack listener and lets connections already established keep flowing for up to +10 seconds, then cuts whatever remains. A request in flight when `stop` is +issued can still complete, at the cost of the command waiting up to that long +longer for shutdown to finish. + Text confirms each successful shutdown and identifies each unavailable owner. JSON and stream-json success data contain `stopped` and `unavailable` ID arrays. A missing default selection reports `found: false`; an unknown explicit name or diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 92146cade9..8663e056af 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -491,6 +491,21 @@ During Starting, acquire the exclusive stack lease, load the instance definition **Orphan sweep.** Stack-labelled containers and session stacks exist only while their lease is held. After readiness, each owner visits every other stack in its state root in the background, with a bounded time per stack. It skips stacks whose lease is held. For a free lease it takes that lease for the duration of the visit, publishing a sweeper record in `owner.json` so clients wait for the visit instead of mistaking it for a starting owner, removes containers labelled with the stack and its data root, and destroys the stack through the owner's own destroy path when its lifetime is `session`. Filtering on the data-root label keeps other state roots untouched. Creating a stack whose identity belongs to a dead session stack reclaims that stack the same way first. +**Abandoned registration.** The orphan sweep above covers a dead owner whose registration survives; +the opposite case is a live owner whose registration disappears out from under it (the state root +is deleted, moved or unmounted). Every owner polls its own `//state.json` on an +interval (30 seconds by default) and treats only a confirmed ENOENT as gone; any other read error, +such as a permission failure or a transient I/O error on an unmounted root, keeps the owner +running and logs a warning instead. On a confirmed-gone registration the owner treats its +ownership as ended: it drains (connection drain, above), stops every workload and removes what it +created through the same registration-independent `cleanupResources` path destroy uses, retrying +transient cleanup failures and giving up only once an engine is confirmed unreachable (the leftover +container ids are logged, not retried forever), releases its port reservations, and exits. It never +reads or writes the registration again, and it does not publish a registration removal: there is +nothing left to update, and any resource claim it already recorded stays available for a future +reconcile. Deletion, a move and an unmount are treated identically, because a stack whose root is +unreachable cannot operate regardless of which of the three caused it. + **Unreachable engine.** An owner of a container stack fails startup with a `runtime-unavailable` reason when its first container sweep finds the engine CLI missing or its daemon not listening; permission, TLS, authentication and timeout failures are ordinary startup errors. `destroy` then proceeds without an owner: it takes the free lease, publishing a sweeper record like the orphan sweep, refuses when any stack data directory cannot be deleted by the current user, removes the host data and the registration with its port claims, and returns the shell commands that remove the stack's containers and engine-volume data once the engine runs. `stop` without a live owner already succeeds without contacting the engine. During Serving, keep the owner alive independently of callers. Sleeping instances still need its public listeners. This is process lifetime management, not automatic service restart or continuous reconciliation. @@ -501,10 +516,21 @@ During Draining: 1. Close admission to new mutations and proxy wake requests. 2. Reject queued work that has not begun. -3. Let executing instance operations settle. -4. Cancel and settle attached commands and stop owned services through their existing operations. -5. For destruction, remove proven-owned data and metadata after shutdown. -6. Send the outcome, close the control endpoint and release ownership. +3. Drain every stack listener, public and dependency alike (connection drain, below). +4. Let executing instance operations settle. +5. Cancel and settle attached commands and stop owned services through their existing operations. +6. For destruction, remove proven-owned data and metadata after shutdown. +7. Send the outcome, close the control endpoint and release ownership. + +**Connection drain.** Stop and destroy share one drain step, run before any service stops: accept +closes on every stack listener, public and internal dependency traffic alike (they are the same +listeners), so no new connection is admitted. Connections already established, including HTTP +keep-alive and a workload's own pooled dependency connections (PostgREST's database pool, for +example), keep flowing until each one closes or a 10 second deadline passes, whichever comes +first. At the deadline every remaining connection is cut. Listener scopes close only afterward, +through the ordinary stop/destroy teardown. Accepted, documented tradeoff: during the drain +window, an admitted request that needs a new dependency connection can fail; the deadline bounds +that window. **Successful whole-stack shutdown.** `stack.stop()` reports success only after admitted work settles, every owned live service and command workload has stopped (including native processes, descendants and containers labeled with this stack and data root), stack listeners close, the shutdown request is acknowledged, and the detached host's exit is confirmed. If workload cleanup cannot be confirmed, stop fails and the live host retains ownership for inspection and retry; the stack is not reported stopped. If cleanup succeeds but host exit cannot be confirmed, stop also fails, though the host may already have exited. A delivered SIGTERM or SIGINT follows this cleanup path. Stop preserves stack definitions, service data, caches and saved port assignments. `destroy` follows the same live-workload cleanup, then removes only proven-owned persistent data. diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index 725c5375cc..2c0eb7efb3 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -1,4 +1,16 @@ -import { Context, Data, Effect, Exit, Layer, Ref, Scope, Semaphore } from "effect"; +import { + Context, + Data, + type Duration, + Effect, + Exit, + Layer, + Ref, + Scope, + Semaphore, + Stream, + SubscriptionRef, +} from "effect"; import { DOCKER_HOST_ALIAS } from "./runtime/Container.ts"; import * as PortReservations from "./namespace/PortReservations.ts"; import * as StackNamespace from "./StackNamespace.ts"; @@ -8,6 +20,16 @@ import { makeHttpProxy, type HttpProxy, type HttpRoute } from "./HttpProxy.ts"; export type NetworkRuntime = "native" | "docker"; +/** The shutdown-drain deadline (F5): established connections keep flowing until this elapses. */ +export const SHUTDOWN_DRAIN_DEADLINE: Duration.Input = "10 seconds"; + +/** The accept/outstanding/cut-all surface every listener kind (`TcpListener`, `HttpProxy`) shares. */ +interface ListenerHandle { + readonly stopAccepting: Effect.Effect; + readonly outstandingConnections: SubscriptionRef.SubscriptionRef; + readonly cutAll: Effect.Effect; +} + type RouteContribution = Pick< HttpRoute, "prefix" | "upstreamPrefix" | "upstreamHost" | "keyRewrite" @@ -67,6 +89,13 @@ export interface Interface { readonly id: string; readonly endpoints: Readonly>; }) => Effect.Effect; + /** + * Shutdown drain (F5): closes accept on every stack listener, public and dependency alike, then + * lets established connections keep flowing until each reaches 0 or `deadline` elapses, whichever + * comes first. At the deadline every remaining connection is cut. Listener scopes are untouched; + * the caller closes them afterward through the ordinary stop/destroy path. + */ + readonly drain: (deadline: Duration.Input) => Effect.Effect; } export class Service extends Context.Service()("@supabase/stack/Network") {} @@ -92,6 +121,23 @@ const makeNetwork = (options: { const owner = yield* Scope.Scope; const gate = yield* Semaphore.make(1); const deferReservations = yield* Ref.make(false); + // Every bound listener (the shared API proxy under "api", every dedicated endpoint under + // "id:name"), tracked only for shutdown drain; registration and deregistration are tied to the + // same scope that owns the listener, so this never drifts from what is actually bound. + const listeners = yield* Ref.make>(new Map()); + const trackListener = (key: string, handle: ListenerHandle) => + Ref.update(listeners, (current) => new Map(current).set(key, handle)).pipe( + Effect.andThen( + Effect.addFinalizer(() => + Ref.update(listeners, (current) => { + if (current.get(key) !== handle) return current; + const next = new Map(current); + next.delete(key); + return next; + }), + ), + ), + ); const shared = yield* Ref.make< | { readonly claim: number; @@ -185,7 +231,9 @@ const makeNetwork = (options: { return { proxy: current.proxy }; } yield* probe(key, host, port); - return { proxy: yield* makeHttpProxy({ host, port }) }; + const proxy = yield* makeHttpProxy({ host, port }); + yield* trackListener(key, proxy); + return { proxy }; } if (endpoint.protocol === "http") { yield* probe(key, host, port); @@ -193,10 +241,12 @@ const makeNetwork = (options: { yield* proxy.setRoutes([ { id, prefix: "/", target: endpoint.backend }, ]); + yield* trackListener(key, proxy); return { proxy }; } yield* probe(key, host, port); const listener = yield* bindTcp(host, port); + yield* trackListener(key, listener); yield* Effect.forkIn( serveTcp(listener, endpoint.backend, `${id}:${name}`).pipe( Effect.provideService(Scope.Scope, endpointScope), @@ -357,12 +407,34 @@ const makeNetwork = (options: { .releaseStack(options.stackId) .pipe(Effect.mapError((cause) => errorFor("release", cause))), ); + // Stopping accepting can only shrink each listener's outstanding count from here on, so waiting + // for every handle to independently reach 0 is equivalent to waiting for the stack-wide total. + const quiesced = (handle: ListenerHandle) => + SubscriptionRef.changes(handle.outstandingConnections).pipe( + Stream.takeUntil((count) => count === 0), + Stream.runDrain, + ); + const drain = Effect.fn("Network.drain")(function* (deadline: Duration.Input) { + const handles = [...(yield* Ref.get(listeners)).values()]; + yield* Effect.forEach(handles, (handle) => handle.stopAccepting, { discard: true }); + const outcome = yield* Effect.forEach(handles, quiesced, { + concurrency: "unbounded", + discard: true, + }).pipe( + Effect.as("quiesced" as const), + Effect.race(Effect.sleep(deadline).pipe(Effect.as("deadline" as const))), + ); + if (outcome === "deadline") + yield* Effect.forEach(handles, (handle) => handle.cutAll, { discard: true }); + yield* Effect.annotateCurrentSpan({ listeners: handles.length, outcome }); + }); return { register, release: release(), beginDestroy: Ref.set(deferReservations, true), cancelDestroy: Ref.set(deferReservations, false), releaseStack: releaseStack(), + drain, } satisfies Interface; }); diff --git a/packages/stack/src/Orchestrator.ts b/packages/stack/src/Orchestrator.ts index 93265bf985..a0c3fdd4c0 100644 --- a/packages/stack/src/Orchestrator.ts +++ b/packages/stack/src/Orchestrator.ts @@ -141,6 +141,8 @@ export interface Interface Effect.Effect; readonly register: (entry: Entry) => Effect.Effect; readonly get: (id: string) => Effect.Effect; + /** A snapshot of every currently registered entry, independent of composition membership. */ + readonly instances: Effect.Effect>; readonly status: (id: string) => Effect.Effect; readonly changes: (id: string) => Stream.Stream; readonly composition: Effect.Effect; @@ -1182,6 +1184,7 @@ export const make = Effect.fn("Orchestrator.make")(function* node(id), + instances: Ref.get(registry).pipe(Effect.map((values) => [...values.values()])), status, changes, composition: Ref.get(composition), diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index e2eca10a57..3825d79c6f 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -1,6 +1,7 @@ import { Context, Crypto, + Duration, Effect, Fiber, FileSystem, @@ -8,6 +9,7 @@ import { Option, Path, Ref, + Schedule, Schema, Scope, Semaphore, @@ -178,6 +180,13 @@ export interface Interface { * containers, claims and saved state. */ readonly destroy: Effect.Effect; + /** + * Ends ownership after a confirmed-gone registration (F6): drains, stops every workload and + * removes what it created through the same registration-independent `cleanupResources` path as + * destroy, then releases port reservations. Never reads or writes the registration, which is + * already gone; a leftover resource past the engine-unreachable backstop is logged, not thrown. + */ + readonly abandon: Effect.Effect; }; readonly setDraining: (draining: boolean) => Effect.Effect; readonly getServing: Effect.Effect; @@ -212,6 +221,44 @@ const serviceError = const rpcError = (operation: string) => Effect.mapError((cause: unknown) => stackError(operation, cause)); +/** A container engine confirmed unreachable (F6): no further retry can make this attempt succeed. */ +const engineUnreachableCleanup = (error: ServiceError) => + error.cause instanceof Container.ContainerError && error.cause.reason === "engine-unavailable"; + +/** Bounded backoff for abandonment cleanup: retries transient failures, gives up on a confirmed-unreachable engine. */ +const abandonCleanupSchedule = Schedule.exponential("200 millis", 2).pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(5))), + ), + Schedule.upTo({ duration: "30 seconds" }), +); + +/** + * Cleans up one abandoned instance through its registration-independent `cleanupResources`, the + * same confirmed path destroy uses. Retries transient failures; an engine confirmed unreachable, or + * retries exhausted, is logged and left behind rather than blocking the owner's exit (F6). + */ +const cleanupAbandonedInstance = (entry: Entry) => + Ref.get(entry.creation).pipe( + Effect.flatMap((config) => + Effect.acquireUseRelease( + Scope.make("parallel"), + (scope) => entry.cleanupResources({ id: entry.id, config, scope }), + (scope, exit) => Scope.close(scope, exit), + ), + ), + Effect.retry({ + schedule: abandonCleanupSchedule, + while: (error) => !engineUnreachableCleanup(error), + }), + Effect.catch((error) => + Effect.logWarning( + `Abandoned stack could not confirm cleanup of ${entry.service} ${entry.id}; leaving it behind`, + error, + ), + ), + ); + const mergeInputs = (creation: ServiceCreation, inputs: Record) => Schema.decodeUnknownEffect(ServiceCreation)({ ...creation, @@ -800,11 +847,16 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { handlers, getStackCredentials, namespace: { - stop: orchestrator.stopNamespace.pipe( - Effect.andThen(sweep), - definitionGate.withPermits(1), - Effect.withSpan("Owner.stopNamespace"), - ), + // Shutdown drain (F5): accept closes on every listener before any service stops, and the + // listener scopes only close afterward, through stop/destroy's ordinary teardown below. + stop: network + .drain(Network.SHUTDOWN_DRAIN_DEADLINE) + .pipe( + Effect.andThen(orchestrator.stopNamespace), + Effect.andThen(sweep), + definitionGate.withPermits(1), + Effect.withSpan("Owner.stopNamespace"), + ), destroy: network.beginDestroy.pipe( // Deferred for the whole of this destroy: every instance's own teardown below closes its // listeners as usual, but leaves its reservation rows for `releaseStack` to drop together, @@ -812,6 +864,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { // immediate per-service deletion again on every exit, success, failure, or interruption // alike, so a service destruction, sweep, or claim-read failure can never leave deferral // stuck on for a stack that is still otherwise live. + Effect.andThen(network.drain(Network.SHUTDOWN_DRAIN_DEADLINE)), Effect.andThen(orchestrator.destroyNamespace), Effect.andThen(sweep), // A claim reconcile deliberately kept (for example one recorded against a different @@ -846,6 +899,35 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { definitionGate.withPermits(1), Effect.withSpan("Owner.destroyNamespace"), ), + // F6: never reads or writes the registration (already confirmed gone), so every step below + // is best-effort and logs rather than fails; the caller must still be able to exit. + abandon: network.drain(Network.SHUTDOWN_DRAIN_DEADLINE).pipe( + Effect.andThen( + orchestrator.stopNamespace.pipe( + Effect.catch((cause) => + Effect.logWarning("Abandoned stack could not confirm every workload stopped", cause), + ), + ), + ), + Effect.andThen( + orchestrator.instances.pipe( + Effect.flatMap((entries) => + Effect.forEach(entries, cleanupAbandonedInstance, { + concurrency: "unbounded", + discard: true, + }), + ), + ), + ), + Effect.andThen( + network.releaseStack.pipe( + Effect.catch((cause) => + Effect.logWarning("Abandoned stack could not release its port reservations", cause), + ), + ), + ), + Effect.withSpan("Owner.abandonNamespace", { attributes: { stack_id: stackId } }), + ), }, setDraining: (value) => Ref.set(draining, value), getServing: Ref.get(draining).pipe(Effect.map((isDraining) => !isDraining)), diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index 2ff53043d4..c9419a8a7c 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -2,6 +2,7 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; import { Context, Crypto, Effect, FileSystem, Layer, Path, Redacted, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { fileURLToPath } from "node:url"; import * as StackNamespace from "./StackNamespace.ts"; import { hasReason, @@ -15,6 +16,10 @@ import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; import { shutdownOwner } from "../tests/owner.ts"; import { noContainerClaims } from "../tests/claims.ts"; +const shortRegistrationPollFixture = fileURLToPath( + new URL("../tests/short-registration-poll-fixture.ts", import.meta.url), +); + const helperImage = "public.ecr.aws/docker/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; @@ -226,3 +231,74 @@ it.live.skipIf(process.platform === "win32")( ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 180_000 }, ); + +it.live.skipIf(process.platform === "win32")( + "removes its containers and exits when its registration is confirmed gone (F6)", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-abandon-docker-" }); + const stackId = `abandon-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const dataRoot = yield* makeDockerDatabaseRoot("stack-abandon-docker-data-", stackId).pipe( + Effect.flatMap(fs.realPath), + ); + const stateRoot = path.dirname(path.dirname(dataRoot)); + const cacheRoot = `${base}/cache`; + const state = yield* stateFor(stateRoot); + yield* state.save({ + id: stackId, + runtime: "docker", + identity: { + projectRoot: `${base}/project`, + branchContext: "abandon-docker-test", + stackName: stackId, + }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + ports: [], + }); + yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); + // The shortened poll interval (F6) comes only from this dedicated test entrypoint, through + // the internal `Context.Reference`; production startup never reads an env var or `Config`. + const access = yield* launchHost(state, { + stateRoot, + cacheRoot, + stackId, + entrypoint: shortRegistrationPollFixture, + }); + const client = yield* ownerClient(access); + const database = yield* client.createService({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("abandon-docker-password"), + jwtSecret: Redacted.make("abandon-docker-jwt-secret-at-least-thirty-two-characters"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* client.startService({ id: database.id }); + yield* client.readyService({ id: database.id }); + expect( + (yield* containers(stackId, dataRoot)).length, + "the owner runs the database container", + ).toBeGreaterThan(0); + + yield* fs.remove(`${stateRoot}/${stackId}/state.json`); + yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.timeout("30 seconds"), + ); + + expect(yield* containers(stackId, dataRoot), "abandonment removes containers").toEqual([]); + expect( + yield* fs.exists(`${stateRoot}/${stackId}/state.json`), + "no registration is republished", + ).toBe(false); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 180_000 }, +); diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts index c3dd274d8e..54b61e3006 100644 --- a/packages/stack/src/StackHost.integration.test.ts +++ b/packages/stack/src/StackHost.integration.test.ts @@ -2,6 +2,7 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; import { Cause, + Clock, Context, Data, Deferred, @@ -1058,3 +1059,130 @@ it.live("destroys a stack only after an abandoned composition settles", () => }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); + +/** A running native database, reachable through its public stack listener like any other client. */ +const withRunningDatabase = Effect.fn("Test.withRunningDatabase")(function* (prefix: string) { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix }); + const state = yield* stateFor(`${root}/state`); + const saved = { + id: "stack", + runtime: "native" as const, + identity: { projectRoot: root, branchContext: "main", stackName: prefix }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + const owner = yield* ownerFor({ + saved, + state, + root: `${root}/data`, + cacheRoot: "/tmp/supabase-stack-artifacts", + }); + const { runtime } = yield* inProcessRuntime(owner, state, root); + const client = yield* ownerClient(runtime.access); + const database = yield* client.createService({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("drain-test-password"), + jwtSecret: Redacted.make("drain-test-jwt-secret-at-least-thirty-two-characters"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* client.startService({ id: database.id }); + yield* client.readyService({ id: database.id }); + const databaseUrl = (yield* client.credentials({ id: database.id, from: "host" })).databaseUrl; + if (databaseUrl === undefined) return yield* Effect.die("Database credentials missing"); + return { runtime, port: Number(new URL(databaseUrl).port) }; +}); + +/** + * Connects a raw socket to `port`; the listener proxies whatever bytes flow without needing the + * real wire protocol, so this alone counts as one established connection for drain purposes. + */ +const openRawSocket = (port: number) => + Effect.acquireRelease( + Effect.callback((resume) => { + const socket = Net.connect(port, "127.0.0.1"); + socket.on("error", () => {}); + socket.once("connect", () => resume(Effect.succeed(socket))); + socket.once("error", (cause) => resume(Effect.fail(hostTestError(cause)))); + return Effect.void; + }), + (socket) => Effect.sync(() => socket.destroy()), + ); + +/** Resolves `true` once a connection to `port` closes on its own shortly after connecting. */ +const isRefusedQuickly = (port: number) => + Effect.scoped( + Effect.gen(function* () { + const socket = yield* openRawSocket(port); + return yield* awaitClosed(socket).pipe( + Effect.as(true), + Effect.race(Effect.sleep("1 second").pipe(Effect.as(false))), + ); + }), + ); + +it.live( + "drains an established connection through a stack listener during stop, and refuses a new one", + () => + Effect.scoped( + Effect.gen(function* () { + const { runtime, port } = yield* withRunningDatabase("stack-host-drain-inflight-"); + const socket = yield* openRawSocket(port); + const shutdown = yield* Effect.forkScoped(shutdownOwner(runtime.access, false)); + // Accept closes synchronously at the start of drain; this bounds the wait for that flip to + // reach the listener, well inside the multi-second window the condition then holds steady. + yield* Effect.sleep("300 millis"); + expect(yield* isRefusedQuickly(port), "a new connection during drain is refused").toBe( + true, + ); + expect(socket.destroyed, "the established connection survives the refusal check").toBe( + false, + ); + // The client finishes on its own, well before the drain deadline. + socket.end(); + yield* awaitClosed(socket).pipe( + Effect.timeoutOrElse({ + duration: "3 seconds", + orElse: () => + Effect.fail(new HostTestError({ message: "Established connection did not close" })), + }), + ); + yield* Fiber.join(shutdown).pipe( + Effect.timeoutOrElse({ + duration: "5 seconds", + orElse: () => + Effect.fail( + new HostTestError({ message: "Stop did not complete once the connection closed" }), + ), + }), + ); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 60_000 }, +); + +it.live( + "cuts a hanging connection at the drain deadline, then stop completes", + () => + Effect.scoped( + Effect.gen(function* () { + const { runtime, port } = yield* withRunningDatabase("stack-host-drain-deadline-"); + const socket = yield* openRawSocket(port); + const startedAt = yield* Clock.currentTimeMillis; + yield* shutdownOwner(runtime.access, false); + const elapsedMillis = (yield* Clock.currentTimeMillis) - startedAt; + // The 10 s drain deadline bounds the wait; generous slack absorbs real scheduling variance. + expect(elapsedMillis).toBeGreaterThanOrEqual(9_000); + expect(elapsedMillis).toBeLessThan(60_000); + expect(socket.destroyed, "the hanging connection is cut at the deadline").toBe(true); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 60_000 }, +); diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index 12b24621ae..bb0fc9697c 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -11,6 +11,7 @@ import { Data, DateTime, Deferred, + type Duration, Effect, Exit, FileSystem, @@ -49,6 +50,15 @@ import { sweepOrphans } from "./Sweep.ts"; import { makeCommandAttachments } from "./host/CommandAttachments.ts"; import * as CommandRunner from "./host/CommandRunner.ts"; +/** + * How often an owner confirms its own registration still exists (F6). Internal only: tests shorten + * it through a dedicated entrypoint that overrides this reference, never through env or `Config`. + */ +export const RegistrationCheckInterval = Context.Reference( + "@supabase/stack/RegistrationCheckInterval", + { defaultValue: () => "30 seconds" }, +); + export interface StackHostOptions { readonly stateRoot: string; readonly cacheRoot: string; @@ -159,6 +169,8 @@ export interface StackHostRuntime { destroy: boolean, response?: ReturnType, ) => Effect.Effect; + /** F6: ends ownership after a confirmed-gone registration; joins an already-running shutdown. */ + readonly abandon: Effect.Effect; readonly exit: Deferred.Deferred; } @@ -328,6 +340,46 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( yield* Fiber.join(fiber); }).pipe(Effect.mapError((cause) => stackError("shutdown", cause))), ); + // F6: a registration-loss poll drives this, never an RPC caller, so there is no response to + // watch and no registration left to touch. An in-flight shutdown already owns `current`; + // abandonment only joins it instead of racing its own cleanup against it. + const abandon = Effect.fn("StackHost.abandon")(() => + gate + .withPermits(1)( + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const existing = yield* Ref.get(current); + if (existing !== undefined) return existing.fiber; + yield* owner.setDraining(true); + const cleanup = restore( + Effect.gen(function* () { + yield* attachments.stopAll; + yield* runner.cleanup; + yield* owner.namespace.abandon; + }), + ).pipe( + // Abandonment must still let the owner exit even if an unexpected defect strikes: + // there is no caller left to report it to and no registration left to retry. + Effect.catchCause((cause) => + Effect.logError("Abandoned stack cleanup failed", cause), + ), + Effect.andThen(closeConnections), + Effect.andThen(Deferred.succeed(exit, undefined)), + Effect.asVoid, + ); + const fiber = yield* Effect.forkIn(cleanup, scope); + yield* Ref.set(current, { destroy: false, fiber }); + return fiber; + }), + ), + ) + .pipe( + Effect.flatMap(Fiber.join), + // `existing.fiber` may be an in-flight normal shutdown, typed to report `StackError`; + // abandonment never fails the caller (the registration poll), so it is logged, not raised. + Effect.catch((cause) => Effect.logError("Stack shutdown failed", cause)), + ), + ); const handlers = StackRpc.of({ ...owner.handlers, runCommand: (input: RunCommandPayload) => attachments.run(input), @@ -377,11 +429,48 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( }); const serve: Effect.Effect = server.serve(application); - return { endpoint: access.endpoint, access, serve, shutdown, closeConnections, exit }; + return { + endpoint: access.endpoint, + access, + serve, + shutdown, + abandon: abandon(), + closeConnections, + exit, + }; }), ); -type HostEvent = "SIGTERM" | "SIGINT" | "creator-gone"; +type HostEvent = "SIGTERM" | "SIGINT" | "creator-gone" | "registration-gone"; + +/** + * Polls for a confirmed-gone registration (F6) and offers `registration-gone` once. Any other + * error (permissions, an unmounted root mid-read) keeps the owner running; only a confirmed ENOENT + * (`state.read` returning `undefined`) counts as abandonment. + */ +const pollRegistration = Effect.fn("StackHost.pollRegistration")(function* ( + state: StackNamespace.Interface, + id: string, + events: Queue.Queue, +) { + const interval = yield* RegistrationCheckInterval; + while (true) { + yield* Effect.sleep(interval); + const saved = yield* state.read(id).pipe( + Effect.tapError((cause) => + Effect.logWarning( + "Could not confirm the stack registration; keeping the owner running", + cause, + ), + ), + Effect.option, + ); + if (Option.isSome(saved) && saved.value === undefined) { + yield* Queue.offer(events, "registration-gone"); + return; + } + } +}); export const runStackHost = Effect.fn("StackHost.run")( (options: StackHostOptions): Effect.Effect => @@ -523,6 +612,7 @@ export const runStackHost = Effect.fn("StackHost.run")( yield* Effect.forkScoped( creatorGone.pipe(Effect.andThen(Queue.offer(events, "creator-gone"))), ); + yield* Effect.forkScoped(pollRegistration(state, id, events)); yield* options.onReady?.(access) ?? Effect.void; yield* Effect.forkScoped( sweepOrphans({ @@ -564,6 +654,10 @@ export const runStackHost = Effect.fn("StackHost.run")( ); break; } + if (event === "registration-gone") { + yield* started.abandon; + break; + } const shutdownSucceeded = yield* started.shutdown(false).pipe( Effect.tapCause((cause) => Effect.logError("Stack shutdown failed", cause)), Effect.matchCause({ onSuccess: () => true, onFailure: () => false }), diff --git a/packages/stack/src/lifetime.integration.test.ts b/packages/stack/src/lifetime.integration.test.ts index 120f0ccb17..ce9e65b1c3 100644 --- a/packages/stack/src/lifetime.integration.test.ts +++ b/packages/stack/src/lifetime.integration.test.ts @@ -16,8 +16,14 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; import { create, open } from "./effect.ts"; +import { launchHost, ownerClient, ownerExitProbe, waitForOwnerExit } from "./HostProcess.ts"; import * as StackNamespace from "./StackNamespace.ts"; -import { assertOwnerExited, captureOwnerPid, watchLeaseRelease } from "../tests/owner.ts"; +import { + assertOwnerExited, + captureOwnerPid, + shutdownOwner, + watchLeaseRelease, +} from "../tests/owner.ts"; class LifetimeTestError extends Data.TaggedError("LifetimeTestError")<{ readonly message: string; @@ -28,6 +34,9 @@ const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; const sessionFixture = fileURLToPath( new URL("../tests/session-client-fixture.ts", import.meta.url), ); +const shortRegistrationPollFixture = fileURLToPath( + new URL("../tests/short-registration-poll-fixture.ts", import.meta.url), +); const stateFor = (root: string) => Layer.build(StackNamespace.layer({ root })).pipe( @@ -220,3 +229,97 @@ it.live("lets only the creating handle start a session stack's owner", () => expect(yield* state.leased(id)).toBe(false); }).pipe(Effect.scoped, Effect.provide(layer)), ); + +it.live.skipIf(process.platform === "win32")( + "exits, stops its native workload and releases its port when its registration is confirmed gone", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-abandon-native-" }); + const stateRoot = `${root}/state`; + const state = yield* stateFor(stateRoot); + const stackId = "abandon-native"; + yield* state.save({ + id: stackId, + runtime: "native", + identity: { projectRoot: root, branchContext: "main", stackName: stackId }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + ports: [], + }); + // The shortened poll interval (F6) comes only from this dedicated test entrypoint, through + // the internal `Context.Reference`; production startup never reads an env var or `Config`. + const access = yield* launchHost(state, { + stateRoot, + cacheRoot, + stackId, + entrypoint: shortRegistrationPollFixture, + }); + const client = yield* ownerClient(access); + const mail = yield* client.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* client.startService({ id: mail.id }); + yield* client.readyService({ id: mail.id }); + const status = yield* client.status({ id: mail.id }); + const port = status.endpoints.find((endpoint) => endpoint.name === "http")?.port; + if (port === undefined) return yield* Effect.die("Missing mail http endpoint"); + const ownerPid = access.endpoint.pid; + const owned = yield* descendantsOf(ownerPid); + expect(owned.length, "the owner runs the native mail service").toBeGreaterThan(0); + + yield* fs.remove(`${stateRoot}/${stackId}/state.json`); + // Detection is bounded by the poll interval (200 ms here); the generous timeout absorbs + // drain, workload stop and cleanup, none of which the default 30 s interval would leave room + // to observe inside a reasonable test budget. + yield* waitForOwnerExit(ownerPid, ownerExitProbe(fs)).pipe(Effect.timeout("15 seconds")); + + expect(owned.filter(alive), "native processes die with their abandoned owner").toEqual([]); + expect( + yield* fs.exists(`${stateRoot}/${stackId}/state.json`), + "no registration is republished", + ).toBe(false); + + // The port reservation is released: a fresh stack can claim the exact same port. + const reclaimedRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-abandon-native-reclaim-", + }); + const reclaimedStateRoot = `${reclaimedRoot}/state`; + const reclaimedState = yield* stateFor(reclaimedStateRoot); + const reclaimedId = "abandon-native-reclaim"; + yield* reclaimedState.save({ + id: reclaimedId, + runtime: "native", + identity: { projectRoot: reclaimedRoot, branchContext: "main", stackName: reclaimedId }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + ports: [], + }); + const reclaimedAccess = yield* launchHost(reclaimedState, { + stateRoot: reclaimedStateRoot, + cacheRoot, + stackId: reclaimedId, + }); + const reclaimedClient = yield* ownerClient(reclaimedAccess); + const reclaimedMail = yield* reclaimedClient.createService({ + service: "mail", + config: {}, + endpoints: { http: { port } }, + }); + yield* reclaimedClient.startService({ id: reclaimedMail.id }); + yield* reclaimedClient.readyService({ id: reclaimedMail.id }); + const reclaimedStatus = yield* reclaimedClient.status({ id: reclaimedMail.id }); + expect(reclaimedStatus.endpoints.find((endpoint) => endpoint.name === "http")?.port).toBe( + port, + ); + yield* shutdownOwner(reclaimedAccess, true); + yield* waitForOwnerExit(reclaimedAccess.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.timeout("15 seconds"), + ); + }).pipe(Effect.scoped, Effect.provide(layer)), + { timeout: 60_000 }, +); diff --git a/packages/stack/tests/short-registration-poll-fixture.ts b/packages/stack/tests/short-registration-poll-fixture.ts new file mode 100644 index 0000000000..2ea68593fb --- /dev/null +++ b/packages/stack/tests/short-registration-poll-fixture.ts @@ -0,0 +1,100 @@ +import { Cause, Effect, Exit, Option, Schema } from "effect"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- readiness is an inherited launcher descriptor. +import { closeSync, writeSync } from "node:fs"; +import { SavedStack } from "../src/StackNamespace.ts"; +import { RegistrationCheckInterval, runStackHost, StackHostError } from "../src/StackHost.ts"; + +/** + * A real owner entrypoint, identical to `internal/host-process.ts`, except it overrides the + * registration-loss poll interval (F6) through the sanctioned internal `Context.Reference` so an + * abandonment test does not wait a real 30 seconds. Production code never reads this from an + * environment variable or `Config`; only this dedicated test fixture supplies a different value. + */ +const writeLine = (value: unknown) => + Effect.gen(function* () { + const serialized = yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))( + value, + ).pipe( + Effect.mapError( + (cause) => new StackHostError({ operation: "ready", message: String(cause), cause }), + ), + ); + yield* Effect.try({ + try: () => writeSync(3, Buffer.from(`${serialized}\n`, "utf8")), + catch: (cause) => new StackHostError({ operation: "ready", message: String(cause), cause }), + }); + }).pipe( + Effect.ensuring( + Effect.sync(() => { + try { + closeSync(3); + } catch { + // The parent may close the inherited descriptor during cancellation. + } + }), + ), + ); + +const [stateRoot, cacheRoot, stackId, register, ...rest] = process.argv.slice(2); + +const program = Effect.gen(function* () { + if ( + stateRoot === undefined || + cacheRoot === undefined || + stackId === undefined || + rest.length > 0 + ) + return yield* new StackHostError({ + operation: "startup", + message: "Expected stateRoot, cacheRoot, stackId and an optional stack to register", + }); + const registered = + register === undefined + ? undefined + : yield* Schema.decodeEffect(Schema.fromJsonString(SavedStack))(register).pipe( + Effect.mapError( + (cause) => new StackHostError({ operation: "startup", message: cause.message }), + ), + ); + let reported = false; + const report = (value: unknown) => + Effect.suspend(() => { + if (reported) return Effect.void; + reported = true; + return writeLine(value); + }); + yield* runStackHost({ + stateRoot, + cacheRoot, + stackId, + ...(registered === undefined ? {} : { register: registered }), + onReady: ({ endpoint, secret }) => report({ type: "ready", endpoint, secret }), + }).pipe( + Effect.catchCause((cause) => { + const failure = Option.getOrUndefined(Cause.findErrorOption(cause)); + return report({ + type: "error", + message: failure?.message ?? Cause.pretty(cause), + ...(failure?.reason === undefined ? {} : { reason: failure.reason }), + }).pipe(Effect.exit, Effect.andThen(Effect.failCause(cause))); + }), + ); +}).pipe(Effect.provideService(RegistrationCheckInterval, "200 millis")); + +const flushed = (stream: NodeJS.WriteStream) => + Effect.callback((resume) => { + stream.write("", () => resume(Effect.void)); + }).pipe(Effect.timeoutOption("1 second")); + +await Effect.runPromise( + program.pipe( + Effect.exit, + Effect.tap((exit) => + Exit.isFailure(exit) + ? Effect.sync(() => process.stderr.write(`${Cause.pretty(exit.cause)}\n`)) + : Effect.void, + ), + Effect.tap(() => Effect.all([flushed(process.stdout), flushed(process.stderr)])), + Effect.flatMap((exit) => Effect.sync(() => process.exit(Exit.isSuccess(exit) ? 0 : 1))), + ), +); From 5ec7da917a75607a4cd9f76891c09bf351bd0b03 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 06:14:09 +0200 Subject: [PATCH 034/142] fix(stack): close review findings in shutdown drain and abandonment - Share destroy's confirmed, serialized execution-lock cleanup (Service.removeData) between destroy and abandonment; registration publication becomes an optional confirm step only destroy passes. - Enter drain under Network's bind gate and reject new binds while draining, so a start admitted mid-drain can never create a listener that escapes the accept cut. - Serialize abandonment with an in-flight shutdown: a joined plain stop still runs abandonment's own resource removal and port release afterward, since stop deliberately preserves both. - Settle each abandonment stage (attachments, command runner, namespace cleanup) independently so one stage's failure never skips the others. - Retry abandonment cleanup until confirmed or the engine is confirmed unreachable (classified by message, matching the shared cleanup path's actual error shapes), with a 2-minute backstop discovered by the whole-root-deletion test to guarantee the owner still exits. - Settle admitted definition work under the owner's definition gate before abandonment drains and removes resources. - Move the shared container-env scratch directory's removal into a registration-independent operation destroy and abandonment both call. - Rework the drain tests to exchange real HTTP application data before and during drain, replacing a fixed sleep with a bounded refusal retry; keep the 10 s deadline on real time with a documented reason. - Rework the abandonment tests to subscribe to the owner's lease release before deleting its registration, and add a whole-state-root deletion test proving cleanup runs from in-memory resources alone. --- packages/stack/src/Network.ts | 16 +- .../src/Orchestrator.integration.test.ts | 2 + packages/stack/src/Orchestrator.ts | 8 +- packages/stack/src/Owner.ts | 75 ++++++--- packages/stack/src/Service.ts | 19 ++- ...ost.container-shutdown.integration.test.ts | 122 +++++++++++++- .../stack/src/StackHost.integration.test.ts | 157 +++++++++++++----- packages/stack/src/StackHost.ts | 42 +++-- .../src/composition/Supabase.unit.test.ts | 1 + .../stack/src/lifetime.integration.test.ts | 12 +- packages/stack/src/namespace/Registry.ts | 12 +- packages/stack/src/runtime/Container.ts | 14 +- packages/stack/tests/standalone-service.ts | 1 + 13 files changed, 371 insertions(+), 110 deletions(-) diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index 2c0eb7efb3..a9caecb6db 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -121,6 +121,11 @@ const makeNetwork = (options: { const owner = yield* Scope.Scope; const gate = yield* Semaphore.make(1); const deferReservations = yield* Ref.make(false); + // Set under `gate` by `drain`, before it snapshots `listeners`; checked under the same gate by + // `bind`, so a bind that hasn't yet acquired the gate when drain starts either completes + // before drain's snapshot (and so is captured by it) or observes this and refuses outright + // (F2) — never slips a new, untracked listener past both the snapshot and the accept cut. + const draining = yield* Ref.make(false); // Every bound listener (the shared API proxy under "api", every dedicated endpoint under // "id:name"), tracked only for shutdown drain; registration and deregistration are tied to the // same scope that owns the listener, so this never drifts from what is actually bound. @@ -208,6 +213,7 @@ const makeNetwork = (options: { Effect.gen(function* () { if (yield* Ref.get(closed)) return yield* errorFor("bind", "Network namespace is closed"); + if (yield* Ref.get(draining)) return yield* errorFor("bind", "Network is draining"); for (const [name, endpoint] of Object.entries(endpoints)) { if ((yield* Ref.get(bound)).has(name)) continue; yield* Effect.uninterruptibleMask((restore) => @@ -415,7 +421,15 @@ const makeNetwork = (options: { Stream.runDrain, ); const drain = Effect.fn("Network.drain")(function* (deadline: Duration.Input) { - const handles = [...(yield* Ref.get(listeners)).values()]; + // Setting `draining` and snapshotting `listeners` atomically under the same gate `bind` + // holds for its whole call (F2) means no bind can complete after this snapshot without + // either being captured by it or observing `draining` and refusing before creating anything. + const handles = yield* gate.withPermits(1)( + Ref.set(draining, true).pipe( + Effect.andThen(Ref.get(listeners)), + Effect.map((current) => [...current.values()]), + ), + ); yield* Effect.forEach(handles, (handle) => handle.stopAccepting, { discard: true }); const outcome = yield* Effect.forEach(handles, quiesced, { concurrency: "unbounded", diff --git a/packages/stack/src/Orchestrator.integration.test.ts b/packages/stack/src/Orchestrator.integration.test.ts index e99cdbfe63..acc31ed3c8 100644 --- a/packages/stack/src/Orchestrator.integration.test.ts +++ b/packages/stack/src/Orchestrator.integration.test.ts @@ -103,6 +103,7 @@ const makeInstance = ( prepare: (inputs, candidate) => configFor(inputs, candidate).pipe(Effect.flatMap((config) => core.prepare(config))), bind: options.bind ?? Ref.set(bound, true), + confirmRemoved: Effect.void, close: orchestrator.status(id).pipe( Effect.flatMap((state) => state.lifecycle === "stopped" && !state.wakeEnabled ? Ref.set(bound, false) : Effect.void, @@ -1250,6 +1251,7 @@ describe("idle races through a real listener", () => { Effect.mapError((cause) => failure(cause.message)), ), close: Effect.void, + confirmRemoved: Effect.void, hasEndpoint: true, inputs: [], outputs: {}, diff --git a/packages/stack/src/Orchestrator.ts b/packages/stack/src/Orchestrator.ts index a0c3fdd4c0..d0019b1971 100644 --- a/packages/stack/src/Orchestrator.ts +++ b/packages/stack/src/Orchestrator.ts @@ -74,6 +74,12 @@ export interface RegisteredInstance { ) => Effect.Effect; readonly bind: Effect.Effect; readonly close: Effect.Effect; + /** + * Publishes this instance's registration removal; run only once `core.removeData`'s resources + * are confirmed removed, as the `confirm` step of that same execution lock. Destroy passes this; + * abandonment never does, because there is no registration left to update. + */ + readonly confirmRemoved: Effect.Effect; readonly hasEndpoint: boolean; readonly inputs: ReadonlyArray; readonly outputs: Readonly>>; @@ -961,7 +967,7 @@ export const make = Effect.fn("Orchestrator.make")(function* Effect.mapError((cause: unknown) => stackError(operation, cause)); -/** A container engine confirmed unreachable (F6): no further retry can make this attempt succeed. */ +/** + * A container engine confirmed unreachable (F5): the shared cleanup path reports this as a plain + * `ServiceError` wrapping whatever the failing operation raised (a `ContainerError` from workload + * removal, or a `DockerDatabaseStorageError` from database volume cleanup, among others), so the + * daemon's own unreachable-daemon phrasing in the message is the one signal common to all of them. + */ const engineUnreachableCleanup = (error: ServiceError) => - error.cause instanceof Container.ContainerError && error.cause.reason === "engine-unavailable"; + Container.engineUnreachableMessage(error.message); -/** Bounded backoff for abandonment cleanup: retries transient failures, gives up on a confirmed-unreachable engine. */ +/** + * Backoff for abandonment cleanup retries: bounded delay, and a generous but finite overall budget. + * Per F6's final decision, cleanup should retry until it is confirmed or the engine is confirmed + * permanently unreachable, not give up merely because a short fixed time budget ran out while the + * engine might still recover — so the engine-unreachable check (below) is the primary, fast give-up + * path. This budget is a backstop, not the intended exit: deleting a stack's whole state root out + * from under a live workload can fail its stop/cleanup for reasons this predicate cannot name (for + * example a mount source that disappeared with it), and an owner that retries such a failure forever + * would violate the one invariant F6 exists to guarantee — that an abandoned owner always exits. + */ const abandonCleanupSchedule = Schedule.exponential("200 millis", 2).pipe( Schedule.modifyDelay(({ duration }) => Effect.succeed(Duration.min(duration, Duration.seconds(5))), ), - Schedule.upTo({ duration: "30 seconds" }), + Schedule.upTo({ duration: "2 minutes" }), ); /** - * Cleans up one abandoned instance through its registration-independent `cleanupResources`, the - * same confirmed path destroy uses. Retries transient failures; an engine confirmed unreachable, or - * retries exhausted, is logged and left behind rather than blocking the owner's exit (F6). + * Cleans up one abandoned instance through the same confirmed, serialized execution path destroy + * uses (`core.removeData`, under the service's execution lock, retrying only steps a previous + * attempt didn't finish), but without a `confirm` step: abandonment never touches the registration. + * Retries transient failures; a confirmed-unreachable engine, or the backstop budget above, is + * logged and left behind, so the owner's exit is never blocked indefinitely (F1, F5). */ const cleanupAbandonedInstance = (entry: Entry) => - Ref.get(entry.creation).pipe( - Effect.flatMap((config) => - Effect.acquireUseRelease( - Scope.make("parallel"), - (scope) => entry.cleanupResources({ id: entry.id, config, scope }), - (scope, exit) => Scope.close(scope, exit), - ), - ), + entry.core.removeData().pipe( Effect.retry({ schedule: abandonCleanupSchedule, while: (error) => !engineUnreachableCleanup(error), @@ -299,8 +308,15 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { const ownerScope = Context.get(services, Scope.Scope); const crypto = Context.get(services, Crypto.Crypto); const path = Context.get(services, Path.Path); + const fs = Context.get(services, FileSystem.FileSystem); const network = yield* Network.Service; const portReservations = yield* PortReservations.Service; + // Shared per-launch environment-file scratch directory (Container.ts), owned by this stack as a + // whole rather than any one instance; registration-independent so destroy and abandonment (F7) + // both reach it without depending on `Registry.remove`, which never runs during abandonment. + const removeContainerEnvRoot = fs + .remove(path.join(options.root, Paths.CONTAINER_ENV_DIRNAME), { recursive: true, force: true }) + .pipe(Effect.mapError(serviceError("cleanup"))); const isPubliclyReserved = (port: number) => portReservations.isReserved(port).pipe( Effect.mapError( @@ -502,12 +518,10 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { Effect.mapError(serviceError("state")), Effect.andThen(recipe.definition.launch(context)), ), - removeData: (context) => - cleanupResources(context).pipe( - Effect.andThen( - removeInstanceRegistration(id).pipe(Effect.mapError(serviceError("state"))), - ), - ), + // Registration-free: `core.removeData` runs this inside its execution lock and, only when + // a `confirm` step is given, publishes the registration removal afterward (destroy passes + // it; abandonment never does). This is the one path both destroy and abandonment share. + removeData: cleanupResources, }, { id, config: initial, report: orchestrator.report }, ).pipe(Effect.provideService(Scope.Scope, ownerScope)); @@ -559,6 +573,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { creation, namespace, cleanupResources, + confirmRemoved: removeInstanceRegistration(id).pipe(Effect.mapError(serviceError("state"))), launch: (generation, inputs, candidate) => configFor(inputs, candidate).pipe( Effect.flatMap((config) => core.launch(generation, config)), @@ -873,7 +888,10 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { Effect.andThen(options.state.readClaims(stackId)), Effect.flatMap((remaining) => remaining.length === 0 - ? network.releaseStack.pipe(Effect.andThen(options.state.remove(stackId))) + ? removeContainerEnvRoot.pipe( + Effect.andThen(network.releaseStack), + Effect.andThen(options.state.remove(stackId)), + ) : Effect.gen(function* () { const currentDaemonId = options.engineTarget?.daemonId; const claimsPath = path.join(options.state.root, stackId, Claims.CLAIMS_FILE); @@ -899,8 +917,9 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { definitionGate.withPermits(1), Effect.withSpan("Owner.destroyNamespace"), ), - // F6: never reads or writes the registration (already confirmed gone), so every step below - // is best-effort and logs rather than fails; the caller must still be able to exit. + // F6: settles admitted definition work under the same gate as stop/destroy first. Never + // reads or writes the registration (already confirmed gone), so every step below is + // best-effort and logs rather than fails; the caller must still be able to exit. abandon: network.drain(Network.SHUTDOWN_DRAIN_DEADLINE).pipe( Effect.andThen( orchestrator.stopNamespace.pipe( @@ -919,6 +938,13 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ), ), ), + Effect.andThen( + removeContainerEnvRoot.pipe( + Effect.catch((cause) => + Effect.logWarning("Abandoned stack could not remove its container-env root", cause), + ), + ), + ), Effect.andThen( network.releaseStack.pipe( Effect.catch((cause) => @@ -926,6 +952,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ), ), ), + definitionGate.withPermits(1), Effect.withSpan("Owner.abandonNamespace", { attributes: { stack_id: stackId } }), ), }, diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index 6a7d3c899e..8f1b9131dd 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -101,8 +101,16 @@ export interface ServiceInstance { readonly storage: ( operation: Effect.Effect, ) => Effect.Effect; - /** Removes the instance's data and marks it unregistered; the service must already be stopped. */ - readonly removeData: Effect.Effect; + /** + * Removes the instance's data and marks it unregistered; the service must already be stopped. + * `confirm`, when given, runs inside the same execution lock once resources are confirmed + * removed and before the instance is marked unregistered — destroy's registration publication. + * Omitting it (abandonment) reuses the identical confirmed, serialized cleanup without touching + * any registration. + */ + readonly removeData: ( + confirm?: Effect.Effect, + ) => Effect.Effect; } interface SessionRecord { @@ -472,7 +480,9 @@ export const makeService = ( ) .pipe(Effect.withSpan("Service.storage", { attributes: { member_id: id } })); - const removeData = Effect.fn("Service.removeData")(function* () { + const removeData = Effect.fn("Service.removeData")(function* ( + confirm: Effect.Effect = Effect.void, + ) { yield* Effect.annotateCurrentSpan({ member_id: id }); yield* execution.withPermit( Effect.gen(function* () { @@ -488,6 +498,7 @@ export const makeService = ( Effect.tapError((error) => update({ error })), Effect.ensuring(Scope.close(dataScope, Exit.void)), ); + yield* confirm; yield* update({ registered: false }); }).pipe(Effect.ensuring(update({ currentOperation: undefined }))); }), @@ -503,6 +514,6 @@ export const makeService = ( stop, reprobe, storage, - removeData: removeData(), + removeData, }; }); diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index c9419a8a7c..fa0d610d37 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -1,6 +1,16 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Context, Crypto, Effect, FileSystem, Layer, Path, Redacted, Stream } from "effect"; +import { + Context, + Crypto, + Effect, + FileSystem, + Layer, + Path, + Redacted, + Schedule, + Stream, +} from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { fileURLToPath } from "node:url"; import * as StackNamespace from "./StackNamespace.ts"; @@ -13,7 +23,7 @@ import { } from "./HostProcess.ts"; import { makeContainerRuntime, resolveEngineTarget } from "./runtime/Container.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; -import { shutdownOwner } from "../tests/owner.ts"; +import { shutdownOwner, watchLeaseRelease } from "../tests/owner.ts"; import { noContainerClaims } from "../tests/claims.ts"; const shortRegistrationPollFixture = fileURLToPath( @@ -287,8 +297,17 @@ it.live.skipIf(process.platform === "win32")( (yield* containers(stackId, dataRoot)).length, "the owner runs the database container", ).toBeGreaterThan(0); + const containerEnvRoot = `${dataRoot}/.container-env`; + expect( + yield* fs.exists(containerEnvRoot), + "the stack's shared container-env scratch directory exists", + ).toBe(true); + // Subscribes to the owner's own exit signal (its lease release) before triggering the + // deletion, rather than polling for it afterward (F9). + const leaseReleased = yield* watchLeaseRelease(stateRoot, stackId); yield* fs.remove(`${stateRoot}/${stackId}/state.json`); + yield* leaseReleased; yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( Effect.timeout("30 seconds"), ); @@ -298,7 +317,106 @@ it.live.skipIf(process.platform === "win32")( yield* fs.exists(`${stateRoot}/${stackId}/state.json`), "no registration is republished", ).toBe(false); + expect( + yield* fs.exists(containerEnvRoot), + "abandonment removes the shared container-env scratch directory (F7)", + ).toBe(false); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 180_000 }, ); + +it.live.skipIf(process.platform === "win32")( + "removes its containers and exits when its whole state root is confirmed gone", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-abandon-docker-root-" }); + const stackId = `abandon-root-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const dataRoot = yield* makeDockerDatabaseRoot( + "stack-abandon-docker-root-data-", + stackId, + ).pipe(Effect.flatMap(fs.realPath)); + const stateRoot = path.dirname(path.dirname(dataRoot)); + const cacheRoot = `${base}/cache`; + const state = yield* stateFor(stateRoot); + yield* state.save({ + id: stackId, + runtime: "docker", + identity: { + projectRoot: `${base}/project`, + branchContext: "abandon-docker-root-test", + stackName: stackId, + }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + ports: [], + }); + yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); + const access = yield* launchHost(state, { + stateRoot, + cacheRoot, + stackId, + entrypoint: shortRegistrationPollFixture, + }); + const client = yield* ownerClient(access); + // `mail` rather than `database`: its container holds no host-mounted data volume, so + // deleting the state root out from under it doesn't also disrupt its own stop path — this + // test is about registration-independent cleanup, not about surviving every workload's + // reaction to losing its mounted storage. + const mail = yield* client.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* client.startService({ id: mail.id }); + yield* client.readyService({ id: mail.id }); + expect( + (yield* containers(stackId, dataRoot)).length, + "the owner runs the mail container", + ).toBeGreaterThan(0); + + // Deletes every discovery-relevant entry in the stack directory, not just `state.json`: the + // claims journal and storage marker are gone too, so cleanup can only come from the owner's + // in-memory resources, never from on-disk discovery. `owner.log` is spared only because it + // is diagnostic output this test reads on failure, never a file the owner itself discovers + // or recovers from. The lease file is removed, so this relies on budgeted polling rather + // than a lease-release subscription. + for (const entry of yield* fs.readDirectory(`${stateRoot}/${stackId}`)) + if (entry !== "owner.log") + yield* fs.remove(`${stateRoot}/${stackId}/${entry}`, { + recursive: true, + force: true, + }); + // Losing the whole root also breaks this workload's own graceful-stop bookkeeping (its + // container-env file is gone too), so its stop retries for real before the owner's 2-minute + // cleanup backstop (F5) gives up on it and still proceeds to remove what it can confirm. + yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.retry({ + schedule: Schedule.spaced("1 second"), + while: (failure) => + failure.reason === "owner-exit-pending" || failure.reason === "owner-exit-zombie", + }), + Effect.timeout("150 seconds"), + ); + + expect( + yield* containers(stackId, dataRoot), + "abandonment removes containers from in-memory resources alone", + ).toEqual([]); + expect( + yield* fs.exists(`${stateRoot}/${stackId}/state.json`), + "no registration is republished", + ).toBe(false); + expect( + yield* fs.exists(`${stateRoot}/${stackId}/claims.json`), + "no claims journal is republished", + ).toBe(false); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 200_000 }, +); diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts index 54b61e3006..e3587511bb 100644 --- a/packages/stack/src/StackHost.integration.test.ts +++ b/packages/stack/src/StackHost.integration.test.ts @@ -13,6 +13,7 @@ import { Layer, Redacted, Ref, + Schedule, Schema, Stream, } from "effect"; @@ -21,6 +22,8 @@ import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"; import * as HttpClient from "effect/unstable/http/HttpClient"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- integration observes exact listener closure. import * as Net from "node:net"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- integration drives a real HTTP keep-alive connection through the public listener. +import * as Http from "node:http"; import { launchHost, ownerAuthorization, @@ -1060,8 +1063,8 @@ it.live("destroys a stack only after an abandoned composition settles", () => ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); -/** A running native database, reachable through its public stack listener like any other client. */ -const withRunningDatabase = Effect.fn("Test.withRunningDatabase")(function* (prefix: string) { +/** A running native mail service, reachable through its public HTTP listener like any client. */ +const withRunningMail = Effect.fn("Test.withRunningMail")(function* (prefix: string) { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix }); const state = yield* stateFor(`${root}/state`); @@ -1083,26 +1086,53 @@ const withRunningDatabase = Effect.fn("Test.withRunningDatabase")(function* (pre }); const { runtime } = yield* inProcessRuntime(owner, state, root); const client = yield* ownerClient(runtime.access); - const database = yield* client.createService({ - service: "database", - config: { - version: "17", - databasePassword: Redacted.make("drain-test-password"), - jwtSecret: Redacted.make("drain-test-jwt-secret-at-least-thirty-two-characters"), - jwtExpiry: 3600, - }, - endpoints: { sql: { port: "auto" } }, + const mail = yield* client.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, }); - yield* client.startService({ id: database.id }); - yield* client.readyService({ id: database.id }); - const databaseUrl = (yield* client.credentials({ id: database.id, from: "host" })).databaseUrl; - if (databaseUrl === undefined) return yield* Effect.die("Database credentials missing"); - return { runtime, port: Number(new URL(databaseUrl).port) }; + yield* client.startService({ id: mail.id }); + yield* client.readyService({ id: mail.id }); + const status = yield* client.status({ id: mail.id }); + const port = status.endpoints.find((endpoint) => endpoint.name === "http")?.port; + if (port === undefined) return yield* Effect.die("Missing mail http endpoint"); + return { runtime, port }; }); /** - * Connects a raw socket to `port`; the listener proxies whatever bytes flow without needing the - * real wire protocol, so this alone counts as one established connection for drain purposes. + * One real HTTP round trip to the public listener over `agent`, proving application data actually + * flows (not just a raw established socket). Resolves once the response ends, with the exact + * socket that carried it, so a caller can later confirm that same connection's fate. + */ +const httpRoundTrip = (port: number, agent: Http.Agent) => + Effect.callback<{ readonly status: number; readonly socket: Net.Socket }, HostTestError>( + (resume) => { + const request = Http.request( + { host: "127.0.0.1", port, path: "/", method: "GET", agent }, + (response) => { + // Captured immediately: the agent may detach `response.socket` once the response ends. + const socket = response.socket; + response.resume(); + response.once("end", () => { + if (socket === null) { + resume(Effect.fail(new HostTestError({ message: "Response has no socket" }))); + return; + } + resume(Effect.succeed({ status: response.statusCode ?? 0, socket })); + }); + }, + ); + request.once("error", (cause) => resume(Effect.fail(hostTestError(cause)))); + request.end(); + return Effect.sync(() => request.destroy()); + }, + ); + +/** + * Connects a raw socket to `port` without speaking HTTP, so Node's own keep-alive idle timeout + * (5 s by default, shorter than the 10 s drain deadline) never applies to it. The listener proxies + * whatever bytes flow without needing a real protocol, so this still counts as one established + * connection for drain purposes. */ const openRawSocket = (port: number) => Effect.acquireRelease( @@ -1116,15 +1146,30 @@ const openRawSocket = (port: number) => (socket) => Effect.sync(() => socket.destroy()), ); -/** Resolves `true` once a connection to `port` closes on its own shortly after connecting. */ -const isRefusedQuickly = (port: number) => - Effect.scoped( - Effect.gen(function* () { - const socket = yield* openRawSocket(port); - return yield* awaitClosed(socket).pipe( - Effect.as(true), - Effect.race(Effect.sleep("1 second").pipe(Effect.as(false))), - ); +/** `true` once a fresh one-shot request to `port` is refused (reset) rather than answered. */ +const isHttpRefused = (port: number) => + Effect.acquireRelease( + Effect.sync(() => new Http.Agent({ keepAlive: false })), + (agent) => Effect.sync(() => agent.destroy()), + ).pipe( + Effect.flatMap((agent) => httpRoundTrip(port, agent)), + Effect.as(false), + Effect.orElseSucceed(() => true), + ); + +/** + * Waits for drain's accept-close to take effect by actively re-checking refusal, instead of + * guessing a fixed delay: the condition, once true, holds for the whole multi-second drain window, + * so a short bounded retry observes it reliably without racing a single arbitrary sleep. + */ +const waitUntilRefused = (port: number) => + Effect.scoped(isHttpRefused(port)).pipe( + Effect.filterOrFail( + (refused) => refused, + () => new HostTestError({ message: "New connections are not refused yet" }), + ), + Effect.retry({ + schedule: Schedule.spaced("50 millis").pipe(Schedule.upTo({ duration: "5 seconds" })), }), ); @@ -1133,27 +1178,32 @@ it.live( () => Effect.scoped( Effect.gen(function* () { - const { runtime, port } = yield* withRunningDatabase("stack-host-drain-inflight-"); - const socket = yield* openRawSocket(port); - const shutdown = yield* Effect.forkScoped(shutdownOwner(runtime.access, false)); - // Accept closes synchronously at the start of drain; this bounds the wait for that flip to - // reach the listener, well inside the multi-second window the condition then holds steady. - yield* Effect.sleep("300 millis"); - expect(yield* isRefusedQuickly(port), "a new connection during drain is refused").toBe( - true, + const { runtime, port } = yield* withRunningMail("stack-host-drain-inflight-"); + const agent = yield* Effect.acquireRelease( + Effect.sync(() => new Http.Agent({ keepAlive: true, maxSockets: 1 })), + (agent) => Effect.sync(() => agent.destroy()), ); - expect(socket.destroyed, "the established connection survives the refusal check").toBe( + const first = yield* httpRoundTrip(port, agent); + expect( + Number.isFinite(first.status), + "the first request gets a real response before drain", + ).toBe(true); + const shutdown = yield* Effect.forkScoped(shutdownOwner(runtime.access, false)); + yield* waitUntilRefused(port); + expect(first.socket.destroyed, "the established connection survives the refusal").toBe( false, ); + const second = yield* httpRoundTrip(port, agent); + expect( + second.socket, + "the second request reuses the same already-established connection", + ).toBe(first.socket); + expect( + Number.isFinite(second.status), + "a request on the established connection still completes during drain", + ).toBe(true); // The client finishes on its own, well before the drain deadline. - socket.end(); - yield* awaitClosed(socket).pipe( - Effect.timeoutOrElse({ - duration: "3 seconds", - orElse: () => - Effect.fail(new HostTestError({ message: "Established connection did not close" })), - }), - ); + agent.destroy(); yield* Fiber.join(shutdown).pipe( Effect.timeoutOrElse({ duration: "5 seconds", @@ -1168,12 +1218,25 @@ it.live( { timeout: 60_000 }, ); +// A real 10 s wait, not `TestClock`: the deadline and this in-process owner's other timers (lease, +// idle, cooldown, readiness polling) share one `Clock` threaded through a single layer composition, +// so virtualizing it risks silently changing those unrelated subsystems for an already-fast (~13 s), +// repeatedly-verified test. it.live( "cuts a hanging connection at the drain deadline, then stop completes", () => Effect.scoped( Effect.gen(function* () { - const { runtime, port } = yield* withRunningDatabase("stack-host-drain-deadline-"); + const { runtime, port } = yield* withRunningMail("stack-host-drain-deadline-"); + // Proves the listener is genuinely functional before testing the hang. A one-shot agent, not + // the hanging connection itself: Node's own ~5 s keep-alive idle timeout would otherwise cut + // an idle HTTP connection before the 10 s drain deadline ever has a chance to. + const proof = yield* Effect.acquireUseRelease( + Effect.sync(() => new Http.Agent({ keepAlive: false })), + (agent) => httpRoundTrip(port, agent), + (agent) => Effect.sync(() => agent.destroy()), + ); + expect(Number.isFinite(proof.status), "the listener answers before the hang").toBe(true); const socket = yield* openRawSocket(port); const startedAt = yield* Clock.currentTimeMillis; yield* shutdownOwner(runtime.access, false); @@ -1181,7 +1244,9 @@ it.live( // The 10 s drain deadline bounds the wait; generous slack absorbs real scheduling variance. expect(elapsedMillis).toBeGreaterThanOrEqual(9_000); expect(elapsedMillis).toBeLessThan(60_000); - expect(socket.destroyed, "the hanging connection is cut at the deadline").toBe(true); + expect(socket.destroyed, "the idle established connection is cut at the deadline").toBe( + true, + ); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 60_000 }, diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index bb0fc9697c..5463ac2e4a 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -343,41 +343,53 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( // F6: a registration-loss poll drives this, never an RPC caller, so there is no response to // watch and no registration left to touch. An in-flight shutdown already owns `current`; // abandonment only joins it instead of racing its own cleanup against it. + // Each stage settles and logs independently: one stage's failure (for example a retained + // native command's termination that cannot be confirmed) must never skip the others (F4). + const settled = (label: string, stage: Effect.Effect) => + stage.pipe( + Effect.catchCause((cause) => Effect.logError(`Abandoned stack: ${label} failed`, cause)), + ); + const abandonResources = settled("abandon namespace cleanup", owner.namespace.abandon); const abandon = Effect.fn("StackHost.abandon")(() => gate .withPermits(1)( Effect.uninterruptibleMask((restore) => Effect.gen(function* () { const existing = yield* Ref.get(current); - if (existing !== undefined) return existing.fiber; + if (existing !== undefined) return existing; yield* owner.setDraining(true); const cleanup = restore( Effect.gen(function* () { - yield* attachments.stopAll; - yield* runner.cleanup; - yield* owner.namespace.abandon; + yield* settled("stopping attachments", attachments.stopAll); + yield* settled("command runner cleanup", runner.cleanup); + yield* abandonResources; }), ).pipe( - // Abandonment must still let the owner exit even if an unexpected defect strikes: - // there is no caller left to report it to and no registration left to retry. - Effect.catchCause((cause) => - Effect.logError("Abandoned stack cleanup failed", cause), - ), Effect.andThen(closeConnections), Effect.andThen(Deferred.succeed(exit, undefined)), Effect.asVoid, ); const fiber = yield* Effect.forkIn(cleanup, scope); - yield* Ref.set(current, { destroy: false, fiber }); - return fiber; + const record = { destroy: false, fiber }; + yield* Ref.set(current, record); + return record; }), ), ) .pipe( - Effect.flatMap(Fiber.join), - // `existing.fiber` may be an in-flight normal shutdown, typed to report `StackError`; - // abandonment never fails the caller (the registration poll), so it is logged, not raised. - Effect.catch((cause) => Effect.logError("Stack shutdown failed", cause)), + Effect.flatMap((record) => + Fiber.join(record.fiber).pipe( + // `record.fiber` may be an in-flight normal shutdown, typed to report `StackError`; + // abandonment never fails the caller (the registration poll), so it is logged. + Effect.catch((cause) => Effect.logError("Stack shutdown failed", cause)), + Effect.andThen( + // A joined plain stop deliberately preserves storage and port reservations; F6 + // cleanup still has to run. A joined destroy already confirmed both, so running + // it again would be redundant (and safe: the shared cleanup path is idempotent). + record.destroy ? Effect.void : abandonResources, + ), + ), + ), ), ); const handlers = StackRpc.of({ diff --git a/packages/stack/src/composition/Supabase.unit.test.ts b/packages/stack/src/composition/Supabase.unit.test.ts index 2e6975f0fe..b5ebab47a0 100644 --- a/packages/stack/src/composition/Supabase.unit.test.ts +++ b/packages/stack/src/composition/Supabase.unit.test.ts @@ -41,6 +41,7 @@ const makeInstance = ( prepare: () => Effect.void, bind: Effect.void, close: Effect.void, + confirmRemoved: Effect.void, hasEndpoint: options.hasEndpoint ?? true, inputs: options.inputs ?? [], outputs: options.outputs ?? {}, diff --git a/packages/stack/src/lifetime.integration.test.ts b/packages/stack/src/lifetime.integration.test.ts index ce9e65b1c3..e37b7e7145 100644 --- a/packages/stack/src/lifetime.integration.test.ts +++ b/packages/stack/src/lifetime.integration.test.ts @@ -271,11 +271,15 @@ it.live.skipIf(process.platform === "win32")( const owned = yield* descendantsOf(ownerPid); expect(owned.length, "the owner runs the native mail service").toBeGreaterThan(0); + // Subscribes to the owner's own exit signal (its lease release, the owner's last act) before + // triggering the deletion, rather than polling for it afterward (F9). + const leaseReleased = yield* watchLeaseRelease(stateRoot, stackId); yield* fs.remove(`${stateRoot}/${stackId}/state.json`); - // Detection is bounded by the poll interval (200 ms here); the generous timeout absorbs - // drain, workload stop and cleanup, none of which the default 30 s interval would leave room - // to observe inside a reasonable test budget. - yield* waitForOwnerExit(ownerPid, ownerExitProbe(fs)).pipe(Effect.timeout("15 seconds")); + yield* leaseReleased; + // The lease is already confirmed released, so the process is already exiting or exited; + // `waitForOwnerExit`'s own internal cap no longer bounds detection, drain and cleanup — only + // this final, now-fast confirmation. + yield* waitForOwnerExit(ownerPid, ownerExitProbe(fs)).pipe(Effect.timeout("10 seconds")); expect(owned.filter(alive), "native processes die with their abandoned owner").toEqual([]); expect( diff --git a/packages/stack/src/namespace/Registry.ts b/packages/stack/src/namespace/Registry.ts index 4ca573809e..b2f5c04988 100644 --- a/packages/stack/src/namespace/Registry.ts +++ b/packages/stack/src/namespace/Registry.ts @@ -8,7 +8,6 @@ import { type NamespaceError, } from "./Capabilities.ts"; import { CLAIMS_FILE } from "./Claims.ts"; -import { CONTAINER_ENV_DIRNAME } from "./Paths.ts"; import * as Publication from "./Publication.ts"; import { removeEmptyDirectory } from "./drivers/FileSystem.ts"; import { acquireLock, isBusy, takeLock } from "./drivers/Sqlite.ts"; @@ -200,14 +199,9 @@ export const make = ( yield* fs .remove(file, { force: true }) .pipe(Effect.mapError((cause) => namespaceError("remove", cause))); - // Namespace-owned scratch space, never user data, so a forced recursive removal here (and - // only here) doesn't mask a service that failed to clean up its own owned root. - yield* fs - .remove(path.join(stackRoot(id), "data", CONTAINER_ENV_DIRNAME), { - recursive: true, - force: true, - }) - .pipe(Effect.mapError((cause) => namespaceError("remove", cause))); + // The container-env scratch directory is removed registration-independently by the owner + // (Owner.ts's `removeContainerEnvRoot`, shared by destroy and abandonment); by the time this + // runs for a confirmed destroy, `data` is already empty of it. yield* removeEmptyDirectory(path.join(stackRoot(id), "data")); yield* removeEmptyDirectory(stackRoot(id)); }); diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index b64649e782..1369359600 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -121,15 +121,21 @@ const transientPullFailure = (error: ContainerError) => /** * Matches an engine CLI that is missing or reports a daemon that is not listening, not one that * rejects the caller. Windows' `error during connect` also wraps authentication and TLS failures, - * so only its refused or missing-endpoint causes match. + * so only its refused or missing-endpoint causes match. Matches regardless of which wrapping error + * type carried the message (a bare `ContainerError`, or another operation's error whose message + * preserves the underlying CLI failure text, such as a database storage cleanup failure). */ +export const engineUnreachableMessage = (message: string): boolean => + /cannot connect to the docker daemon|connection refused|connect: no such file or directory|error during connect:[^\n]*(?:docker daemon is not running|the system cannot find the file specified)/iu.test( + message, + ); + +/** Also matches a missing engine CLI binary, which never produces a message to test. */ export const engineUnreachable = (error: ContainerError) => (error.cause instanceof PlatformError.PlatformError && error.cause.reason._tag === "NotFound" && error.cause.reason.method === "spawn") || - /cannot connect to the docker daemon|connection refused|connect: no such file or directory|error during connect:[^\n]*(?:docker daemon is not running|the system cannot find the file specified)/iu.test( - error.message, - ); + engineUnreachableMessage(error.message); /** Runs one engine CLI invocation outside any pinned target, for resolving that target itself. */ const runRaw = ( diff --git a/packages/stack/tests/standalone-service.ts b/packages/stack/tests/standalone-service.ts index 131f39709b..39d8ff6640 100644 --- a/packages/stack/tests/standalone-service.ts +++ b/packages/stack/tests/standalone-service.ts @@ -34,6 +34,7 @@ export const makeStandaloneService = Effect.fn("Test.makeStandaloneService")(fun prepare: (_inputs, restarted) => configFor(restarted).pipe(Effect.flatMap(core.prepare)), bind: Effect.void, close: Effect.void, + confirmRemoved: Effect.void, hasEndpoint: true, inputs: [], outputs: {}, From 2844c978efe7a393f4ce70e9f1a905fc1a9989b4 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 06:48:58 +0200 Subject: [PATCH 035/142] fix(stack): fix scoped temp-dir cleanup panic and classify engine errors by type Ordinary stop crashed with an unrecoverable defect once a stack's whole state root was already deleted, because the platform's scoped temp directory cleanup dies on ENOENT without force-removing; switch to an unscoped temp directory with an explicit force-remove finalizer so cleanup converges instead of relying on an elapsed-time give-up, which is now removed. Classify engine-unreachable failures with a typed reason carried through DockerDatabaseStorageError instead of matching on message text downstream. Make the shutdown drain deadline an internal Context.Reference so tests can inject a short deadline and assert cut ordering by observation instead of a wall-clock bound. --- packages/stack/src/Network.ts | 24 ++++++---- packages/stack/src/Owner.ts | 46 ++++++++++--------- ...ost.container-shutdown.integration.test.ts | 7 +-- .../stack/src/StackHost.integration.test.ts | 44 +++++++++++------- packages/stack/src/runtime/Container.ts | 10 +++- .../src/storage/DockerDatabaseStorage.ts | 21 ++++++++- 6 files changed, 98 insertions(+), 54 deletions(-) diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index a9caecb6db..7a1606c094 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -20,8 +20,15 @@ import { makeHttpProxy, type HttpProxy, type HttpRoute } from "./HttpProxy.ts"; export type NetworkRuntime = "native" | "docker"; -/** The shutdown-drain deadline (F5): established connections keep flowing until this elapses. */ -export const SHUTDOWN_DRAIN_DEADLINE: Duration.Input = "10 seconds"; +/** + * The shutdown-drain deadline (F5): established connections keep flowing until this elapses. + * Internal only, following the `RegistrationCheckInterval` pattern: tests shorten it through this + * reference; production never reads it from an environment variable or `Config`. + */ +export const ShutdownDrainDeadline = Context.Reference( + "@supabase/stack/ShutdownDrainDeadline", + { defaultValue: () => "10 seconds" }, +); /** The accept/outstanding/cut-all surface every listener kind (`TcpListener`, `HttpProxy`) shares. */ interface ListenerHandle { @@ -91,11 +98,11 @@ export interface Interface { }) => Effect.Effect; /** * Shutdown drain (F5): closes accept on every stack listener, public and dependency alike, then - * lets established connections keep flowing until each reaches 0 or `deadline` elapses, whichever - * comes first. At the deadline every remaining connection is cut. Listener scopes are untouched; - * the caller closes them afterward through the ordinary stop/destroy path. + * lets established connections keep flowing until each reaches 0 or `ShutdownDrainDeadline` + * elapses, whichever comes first. At the deadline every remaining connection is cut. Listener + * scopes are untouched; the caller closes them afterward through the ordinary stop/destroy path. */ - readonly drain: (deadline: Duration.Input) => Effect.Effect; + readonly drain: Effect.Effect; } export class Service extends Context.Service()("@supabase/stack/Network") {} @@ -420,7 +427,8 @@ const makeNetwork = (options: { Stream.takeUntil((count) => count === 0), Stream.runDrain, ); - const drain = Effect.fn("Network.drain")(function* (deadline: Duration.Input) { + const drain = Effect.fn("Network.drain")(function* () { + const deadline = yield* ShutdownDrainDeadline; // Setting `draining` and snapshotting `listeners` atomically under the same gate `bind` // holds for its whole call (F2) means no bind can complete after this snapshot without // either being captured by it or observing `draining` and refusing before creating anything. @@ -448,7 +456,7 @@ const makeNetwork = (options: { beginDestroy: Ref.set(deferReservations, true), cancelDestroy: Ref.set(deferReservations, false), releaseStack: releaseStack(), - drain, + drain: drain(), } satisfies Interface; }); diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index 3280be5518..7873ed6a56 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -65,6 +65,7 @@ import { } from "./services/Catalog.ts"; import type { CatalogError } from "./services/Recipe.ts"; import * as Container from "./runtime/Container.ts"; +import { DockerDatabaseStorageError } from "./storage/DockerDatabaseStorage.ts"; import * as Claims from "./namespace/Claims.ts"; import { namespaceError } from "./namespace/Capabilities.ts"; import { lstatPath } from "./namespace/drivers/FileSystem.ts"; @@ -227,32 +228,35 @@ const rpcError = (operation: string) => * removal, or a `DockerDatabaseStorageError` from database volume cleanup, among others), so the * daemon's own unreachable-daemon phrasing in the message is the one signal common to all of them. */ +/** + * A container engine confirmed unreachable (F5), checked as a typed reason through the cause + * chain rather than by matching message text: a direct `ContainerError` from workload removal, or + * a `DockerDatabaseStorageError` from database volume cleanup, whose own `reason` field is set + * once, at the raw engine CLI invocation that is the only place either module can observe it. + */ const engineUnreachableCleanup = (error: ServiceError) => - Container.engineUnreachableMessage(error.message); + (error.cause instanceof Container.ContainerError && + error.cause.reason === "engine-unavailable") || + (Schema.is(DockerDatabaseStorageError)(error.cause) && + error.cause.reason === "engine-unavailable"); /** - * Backoff for abandonment cleanup retries: bounded delay, and a generous but finite overall budget. - * Per F6's final decision, cleanup should retry until it is confirmed or the engine is confirmed - * permanently unreachable, not give up merely because a short fixed time budget ran out while the - * engine might still recover — so the engine-unreachable check (below) is the primary, fast give-up - * path. This budget is a backstop, not the intended exit: deleting a stack's whole state root out - * from under a live workload can fail its stop/cleanup for reasons this predicate cannot name (for - * example a mount source that disappeared with it), and an owner that retries such a failure forever - * would violate the one invariant F6 exists to guarantee — that an abandoned owner always exits. + * Backoff for abandonment cleanup retries: bounded delay, unbounded attempts. Per F6's final + * decision, cleanup retries until it is confirmed or the engine is confirmed permanently + * unreachable — never until a fixed time budget runs out while the engine might still recover. */ const abandonCleanupSchedule = Schedule.exponential("200 millis", 2).pipe( Schedule.modifyDelay(({ duration }) => Effect.succeed(Duration.min(duration, Duration.seconds(5))), ), - Schedule.upTo({ duration: "2 minutes" }), ); /** * Cleans up one abandoned instance through the same confirmed, serialized execution path destroy * uses (`core.removeData`, under the service's execution lock, retrying only steps a previous * attempt didn't finish), but without a `confirm` step: abandonment never touches the registration. - * Retries transient failures; a confirmed-unreachable engine, or the backstop budget above, is - * logged and left behind, so the owner's exit is never blocked indefinitely (F1, F5). + * Retries transient failures; only a confirmed-unreachable engine is logged and left behind, so a + * recoverable failure never blocks the owner's exit (F1, F5). */ const cleanupAbandonedInstance = (entry: Entry) => entry.core.removeData().pipe( @@ -864,14 +868,12 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { namespace: { // Shutdown drain (F5): accept closes on every listener before any service stops, and the // listener scopes only close afterward, through stop/destroy's ordinary teardown below. - stop: network - .drain(Network.SHUTDOWN_DRAIN_DEADLINE) - .pipe( - Effect.andThen(orchestrator.stopNamespace), - Effect.andThen(sweep), - definitionGate.withPermits(1), - Effect.withSpan("Owner.stopNamespace"), - ), + stop: network.drain.pipe( + Effect.andThen(orchestrator.stopNamespace), + Effect.andThen(sweep), + definitionGate.withPermits(1), + Effect.withSpan("Owner.stopNamespace"), + ), destroy: network.beginDestroy.pipe( // Deferred for the whole of this destroy: every instance's own teardown below closes its // listeners as usual, but leaves its reservation rows for `releaseStack` to drop together, @@ -879,7 +881,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { // immediate per-service deletion again on every exit, success, failure, or interruption // alike, so a service destruction, sweep, or claim-read failure can never leave deferral // stuck on for a stack that is still otherwise live. - Effect.andThen(network.drain(Network.SHUTDOWN_DRAIN_DEADLINE)), + Effect.andThen(network.drain), Effect.andThen(orchestrator.destroyNamespace), Effect.andThen(sweep), // A claim reconcile deliberately kept (for example one recorded against a different @@ -920,7 +922,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { // F6: settles admitted definition work under the same gate as stop/destroy first. Never // reads or writes the registration (already confirmed gone), so every step below is // best-effort and logs rather than fails; the caller must still be able to exit. - abandon: network.drain(Network.SHUTDOWN_DRAIN_DEADLINE).pipe( + abandon: network.drain.pipe( Effect.andThen( orchestrator.stopNamespace.pipe( Effect.catch((cause) => diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index fa0d610d37..17f2ed8d02 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -392,16 +392,13 @@ it.live.skipIf(process.platform === "win32")( recursive: true, force: true, }); - // Losing the whole root also breaks this workload's own graceful-stop bookkeeping (its - // container-env file is gone too), so its stop retries for real before the owner's 2-minute - // cleanup backstop (F5) gives up on it and still proceeds to remove what it can confirm. yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( Effect.retry({ schedule: Schedule.spaced("1 second"), while: (failure) => failure.reason === "owner-exit-pending" || failure.reason === "owner-exit-zombie", }), - Effect.timeout("150 seconds"), + Effect.timeout("30 seconds"), ); expect( @@ -418,5 +415,5 @@ it.live.skipIf(process.platform === "win32")( ).toBe(false); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), - { timeout: 200_000 }, + { timeout: 180_000 }, ); diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts index e3587511bb..13bac03c25 100644 --- a/packages/stack/src/StackHost.integration.test.ts +++ b/packages/stack/src/StackHost.integration.test.ts @@ -2,7 +2,6 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; import { Cause, - Clock, Context, Data, Deferred, @@ -32,6 +31,7 @@ import { waitForOwnerExit, type HostAccess, } from "./HostProcess.ts"; +import * as Network from "./Network.ts"; import * as Owner from "./Owner.ts"; import { OrchestratorError } from "./Orchestrator.ts"; import { CommandEvent, StackError } from "./Rpc.ts"; @@ -1218,10 +1218,11 @@ it.live( { timeout: 60_000 }, ); -// A real 10 s wait, not `TestClock`: the deadline and this in-process owner's other timers (lease, -// idle, cooldown, readiness polling) share one `Clock` threaded through a single layer composition, -// so virtualizing it risks silently changing those unrelated subsystems for an already-fast (~13 s), -// repeatedly-verified test. +// Drives the deadline with a shortened `Network.ShutdownDrainDeadline` (the internal reference +// production always defaults to 10 s) rather than asserting a wall-clock lower bound: a timed +// assertion only proves this test's own timer started before drain's, which a loaded CI runner can +// easily violate without the drain logic itself being wrong. Ordering is observed instead: the +// connection is confirmed still open while draining, then confirmed cut once stop completes. it.live( "cuts a hanging connection at the drain deadline, then stop completes", () => @@ -1230,7 +1231,7 @@ it.live( const { runtime, port } = yield* withRunningMail("stack-host-drain-deadline-"); // Proves the listener is genuinely functional before testing the hang. A one-shot agent, not // the hanging connection itself: Node's own ~5 s keep-alive idle timeout would otherwise cut - // an idle HTTP connection before the 10 s drain deadline ever has a chance to. + // an idle HTTP connection before even a shortened drain deadline has a chance to. const proof = yield* Effect.acquireUseRelease( Effect.sync(() => new Http.Agent({ keepAlive: false })), (agent) => httpRoundTrip(port, agent), @@ -1238,16 +1239,27 @@ it.live( ); expect(Number.isFinite(proof.status), "the listener answers before the hang").toBe(true); const socket = yield* openRawSocket(port); - const startedAt = yield* Clock.currentTimeMillis; - yield* shutdownOwner(runtime.access, false); - const elapsedMillis = (yield* Clock.currentTimeMillis) - startedAt; - // The 10 s drain deadline bounds the wait; generous slack absorbs real scheduling variance. - expect(elapsedMillis).toBeGreaterThanOrEqual(9_000); - expect(elapsedMillis).toBeLessThan(60_000); - expect(socket.destroyed, "the idle established connection is cut at the deadline").toBe( - true, + const shutdown = yield* Effect.forkScoped(shutdownOwner(runtime.access, false)); + // Observed, not timed: confirms drain has started (new connections already refused) while + // the hanging connection is still deliberately held open, ahead of the deadline cut. + yield* waitUntilRefused(port); + expect(socket.destroyed, "the established connection survives into the drain window").toBe( + false, + ); + yield* Fiber.join(shutdown).pipe( + Effect.timeoutOrElse({ + duration: "10 seconds", + orElse: () => + Effect.fail( + new HostTestError({ message: "Stop did not complete after the drain deadline" }), + ), + }), ); + expect(socket.destroyed, "the hanging connection is cut at the deadline").toBe(true); }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), - { timeout: 60_000 }, + ).pipe( + Effect.provideService(Network.ShutdownDrainDeadline, "2 seconds"), + Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp)), + ), + { timeout: 30_000 }, ); diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index 1369359600..91b759e4e4 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -642,9 +642,17 @@ export const makeContainerRuntime = (options: { yield* fs .makeDirectory(containerEnvRoot, { recursive: true, mode: 0o700 }) .pipe(Effect.mapError((cause) => errorFor("environment", cause))); + // Not `makeTempDirectoryScoped`: its scoped cleanup removes without `force`, then dies on + // any failure, including a confirmed ENOENT when this stack's whole root (this directory's + // owned ancestor) is already gone — turning an ordinary stop into an unrecoverable defect + // (F5/F1 root cause). A plain temp directory plus a tolerant finalizer treats "already gone" + // as already cleaned up, matching this directory's own owned-and-disposable nature. const directory = yield* fs - .makeTempDirectoryScoped({ directory: containerEnvRoot, prefix: "container-" }) + .makeTempDirectory({ directory: containerEnvRoot, prefix: "container-" }) .pipe(Effect.mapError((cause) => errorFor("environment", cause))); + yield* Effect.addFinalizer(() => + fs.remove(directory, { recursive: true, force: true }).pipe(Effect.ignore), + ); const envPath = path.join(directory, "environment"); yield* fs .writeFileString( diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index 8c126db241..a9fb7fc87c 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -18,7 +18,11 @@ import { failureMessage } from "../internal/failure-message.ts"; import { testRunLabelArgs as readTestRunLabelArgs } from "../internal/test-run-label.ts"; import type * as Claims from "../namespace/Claims.ts"; import * as Publication from "../namespace/Publication.ts"; -import type { ContainerRuntime, EngineTarget } from "../runtime/Container.ts"; +import { + engineUnreachableMessage, + type ContainerRuntime, + type EngineTarget, +} from "../runtime/Container.ts"; import { composeProjectFor } from "../runtime/ContainerName.ts"; import type { DatabaseRuntime } from "../services/Database.ts"; import { @@ -57,7 +61,13 @@ interface ResolvedDaemon extends DaemonIdentity { export class DockerDatabaseStorageError extends Schema.TaggedError()( "DockerDatabaseStorageError", - { operation: Schema.String, message: Schema.String, cause: Schema.optionalKey(Schema.Defect()) }, + { + operation: Schema.String, + message: Schema.String, + cause: Schema.optionalKey(Schema.Defect()), + /** Set once, at the raw engine CLI invocation that produced `message` (see `errorFor`). */ + reason: Schema.optionalKey(Schema.Literal("engine-unavailable")), + }, ) {} export interface DatabaseStorageMount { @@ -90,6 +100,9 @@ export interface DockerDatabaseStorage { } // Snapshot failures keep the protocol's operation so both engines report the same step. +// The engine CLI's own stderr text is the only unreachable-daemon signal this module ever sees +// (it invokes the raw CLI directly, not through `runtime/Container.ts`'s typed `ContainerError`), +// so the message is classified here, once, at the source, and carried onward as `reason`. const errorFor = (operation: string, cause: unknown) => Schema.is(DockerDatabaseStorageError)(cause) ? cause @@ -98,11 +111,15 @@ const errorFor = (operation: string, cause: unknown) => operation: cause.operation, message: cause.message, cause, + ...(engineUnreachableMessage(cause.message) ? { reason: "engine-unavailable" } : {}), }) : new DockerDatabaseStorageError({ operation, message: failureMessage(cause), cause, + ...(engineUnreachableMessage(failureMessage(cause)) + ? { reason: "engine-unavailable" } + : {}), }); const parseMajor = (version: string): number | undefined => { From 408b55f1dfb5a0de2fa82b415747b7c035d0d107 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 08:24:16 +0200 Subject: [PATCH 036/142] fix(stack): stop abandonment from touching port reservations and fix drain/cleanup recovery gaps Abandonment never releases port reservations now: a stale one stays for Ports.ts's own lazy reclamation once the holder's registration is confirmed gone, which is exactly the abandonment trigger, so it never needs registration access even once the whole state root is gone. Report confirmed termination to the lifecycle reducer whenever removeData cleans up a session a failed stop retained, or it stays Stopping forever. Classify an unavailable engine once, at every invocation boundary, and preserve that reason through wrapping instead of matching message text downstream. Recover shutdown-failure-after- drain by resetting network admission and listener accept together with the owner's own recovery. Retry retained native-command cleanup and scratch-directory removal under the same policy as namespace cleanup before abandonment permits the owner to exit, and inspect the joined shutdown outcome rather than its requested mode before skipping that cleanup. Protect acquiring the container-env scratch directory and installing its removal finalizer together. Rewrite the drain tests to establish progress on the exact connection under test and reject a proxy-generated response. --- packages/stack/src/HttpProxy.ts | 5 + packages/stack/src/Network.ts | 22 +++ .../src/Orchestrator.integration.test.ts | 2 + packages/stack/src/Orchestrator.ts | 13 +- packages/stack/src/Owner.ts | 51 +++---- packages/stack/src/Proxy.ts | 5 + .../stack/src/Service.integration.test.ts | 42 ++++++ packages/stack/src/Service.ts | 16 ++- ...ost.container-shutdown.integration.test.ts | 67 ++++++--- .../stack/src/StackHost.integration.test.ts | 128 ++++++++++++------ packages/stack/src/StackHost.ts | 34 ++++- .../src/composition/Supabase.unit.test.ts | 1 + .../src/runtime/Container.integration.test.ts | 58 ++++++++ packages/stack/src/runtime/Container.ts | 89 ++++++------ .../src/storage/DockerDatabaseStorage.ts | 13 +- packages/stack/tests/standalone-service.ts | 1 + 16 files changed, 407 insertions(+), 140 deletions(-) diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index c896997818..1348849514 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -47,6 +47,8 @@ export interface HttpProxy { readonly setRoutes: (routes: ReadonlyArray) => Effect.Effect; /** Stops accepting new connections on the listener; established connections keep flowing. */ readonly stopAccepting: Effect.Effect; + /** Reverses `stopAccepting` (F6): recovers a listener left refusing after a failed drain. */ + readonly resumeAccepting: Effect.Effect; /** The count of established client connections, observable until it reaches 0. */ readonly outstandingConnections: SubscriptionRef.SubscriptionRef; /** Destroys every established connection immediately. */ @@ -531,6 +533,9 @@ export const makeHttpProxy = (options: { stopAccepting: Effect.sync(() => { accepting = false; }), + resumeAccepting: Effect.sync(() => { + accepting = true; + }), outstandingConnections, cutAll: Effect.sync(() => { for (const socket of sockets) socket.destroy(); diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index 7a1606c094..8b44a93951 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -33,6 +33,7 @@ export const ShutdownDrainDeadline = Context.Reference( /** The accept/outstanding/cut-all surface every listener kind (`TcpListener`, `HttpProxy`) shares. */ interface ListenerHandle { readonly stopAccepting: Effect.Effect; + readonly resumeAccepting: Effect.Effect; readonly outstandingConnections: SubscriptionRef.SubscriptionRef; readonly cutAll: Effect.Effect; } @@ -103,6 +104,14 @@ export interface Interface { * scopes are untouched; the caller closes them afterward through the ordinary stop/destroy path. */ readonly drain: Effect.Effect; + /** + * Recovers from a shutdown that failed after {@link drain} began (F6): resets the drain flag + * `bind` checks and resumes accepting on every currently tracked listener, so a start/restart + * that follows a failed shutdown binds normally instead of refusing as still draining forever. + * Paired with the owner's own admission recovery; never called while a shutdown might still + * confirm success on its own. + */ + readonly recoverDrain: Effect.Effect; } export class Service extends Context.Service()("@supabase/stack/Network") {} @@ -450,6 +459,18 @@ const makeNetwork = (options: { yield* Effect.forEach(handles, (handle) => handle.cutAll, { discard: true }); yield* Effect.annotateCurrentSpan({ listeners: handles.length, outcome }); }); + const recoverDrain = Effect.fn("Network.recoverDrain")(() => + gate.withPermits(1)( + Ref.set(draining, false).pipe( + Effect.andThen(Ref.get(listeners)), + Effect.flatMap((current) => + Effect.forEach([...current.values()], (handle) => handle.resumeAccepting, { + discard: true, + }), + ), + ), + ), + ); return { register, release: release(), @@ -457,6 +478,7 @@ const makeNetwork = (options: { cancelDestroy: Ref.set(deferReservations, false), releaseStack: releaseStack(), drain: drain(), + recoverDrain: recoverDrain(), } satisfies Interface; }); diff --git a/packages/stack/src/Orchestrator.integration.test.ts b/packages/stack/src/Orchestrator.integration.test.ts index acc31ed3c8..3b4ef1fda6 100644 --- a/packages/stack/src/Orchestrator.integration.test.ts +++ b/packages/stack/src/Orchestrator.integration.test.ts @@ -104,6 +104,7 @@ const makeInstance = ( configFor(inputs, candidate).pipe(Effect.flatMap((config) => core.prepare(config))), bind: options.bind ?? Ref.set(bound, true), confirmRemoved: Effect.void, + release: Effect.void, close: orchestrator.status(id).pipe( Effect.flatMap((state) => state.lifecycle === "stopped" && !state.wakeEnabled ? Ref.set(bound, false) : Effect.void, @@ -1252,6 +1253,7 @@ describe("idle races through a real listener", () => { ), close: Effect.void, confirmRemoved: Effect.void, + release: Effect.void, hasEndpoint: true, inputs: [], outputs: {}, diff --git a/packages/stack/src/Orchestrator.ts b/packages/stack/src/Orchestrator.ts index d0019b1971..bec2cf36c0 100644 --- a/packages/stack/src/Orchestrator.ts +++ b/packages/stack/src/Orchestrator.ts @@ -80,6 +80,13 @@ export interface RegisteredInstance { * abandonment never does, because there is no registration left to update. */ readonly confirmRemoved: Effect.Effect; + /** + * Releases this instance's network reservations once its data is confirmed removed: destroy's + * own step, run right after `core.removeData` succeeds. Abandonment never calls it — a stale + * reservation stays for `Ports.ts`'s lazy reclamation once the holder's registration is + * confirmed gone, exactly the abandonment trigger, so abandonment has nothing to release itself. + */ + readonly release: Effect.Effect; readonly hasEndpoint: boolean; readonly inputs: ReadonlyArray; readonly outputs: Readonly>>; @@ -967,7 +974,11 @@ export const make = Effect.fn("Orchestrator.make")(function* ; + /** + * Recovers network admission after a shutdown that failed once drain had already begun (F6): + * see {@link Network.Interface.recoverDrain}. + */ + readonly recoverDraining: Effect.Effect; }; readonly setDraining: (draining: boolean) => Effect.Effect; readonly getServing: Effect.Effect; @@ -244,8 +251,10 @@ const engineUnreachableCleanup = (error: ServiceError) => * Backoff for abandonment cleanup retries: bounded delay, unbounded attempts. Per F6's final * decision, cleanup retries until it is confirmed or the engine is confirmed permanently * unreachable — never until a fixed time budget runs out while the engine might still recover. + * Exported so every abandonment cleanup stage shares one retry policy, including `StackHost.ts`'s + * retained native-command cleanup, which is outside this module's own cleanup paths. */ -const abandonCleanupSchedule = Schedule.exponential("200 millis", 2).pipe( +export const abandonCleanupSchedule = Schedule.exponential("200 millis", 2).pipe( Schedule.modifyDelay(({ duration }) => Effect.succeed(Duration.min(duration, Duration.seconds(5))), ), @@ -499,21 +508,14 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { }); const initial = recipe.creation; const creation = yield* Ref.make(initial); - const namespaceRef = yield* Ref.make(undefined); // Independent of the saved registration and of on-disk discovery: removes containers and the - // owned storage namespace, then releases the network namespace. Destroy below runs this to - // completion before publishing the registration removal, so a confirmed removal is never - // recorded ahead of the resources it describes; a future abandonment path can call this same - // operation and release the stack's ports without ever reading or saving the registration. + // owned storage namespace. Shared as-is by destroy and abandonment, through `core.removeData`; + // neither reads or writes the registration. Network reservations are never part of this: they + // are destroy's own separate `release` step (`Orchestrator.destroy`), never abandonment's, + // since a stale reservation stays for `Ports.ts`'s lazy reclamation once the holder's + // registration is confirmed gone — exactly the abandonment trigger. const cleanupResources = (context: ServiceInstanceContext) => - recipe.definition.removeData(context).pipe( - Effect.andThen( - Ref.get(namespaceRef).pipe( - Effect.flatMap((namespace) => namespace?.release ?? Effect.void), - Effect.mapError(serviceError("release")), - ), - ), - ); + recipe.definition.removeData(context); const core = yield* makeService( { ...recipe.definition, @@ -568,7 +570,6 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { }), ); const namespace = yield* network.register({ id, endpoints }); - yield* Ref.set(namespaceRef, namespace); const entry: Entry = { id, service: initial.service, @@ -578,6 +579,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { namespace, cleanupResources, confirmRemoved: removeInstanceRegistration(id).pipe(Effect.mapError(serviceError("state"))), + release: namespace.release.pipe(Effect.mapError(serviceError("release"))), launch: (generation, inputs, candidate) => configFor(inputs, candidate).pipe( Effect.flatMap((config) => core.launch(generation, config)), @@ -940,23 +942,24 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ), ), ), + // F7: retried until confirmed, like every other abandonment cleanup stage; its own + // `force: true` only tolerates an already-gone directory, never a transient failure. + // Port reservations are never released here (simplification over F2/F7): a stale + // reservation stays for `Ports.ts`'s lazy reclamation once the holder's registration is + // confirmed gone — exactly the abandonment trigger — so abandonment never touches + // `Ports.ts`'s registry at all, even once the state root itself no longer exists. Effect.andThen( removeContainerEnvRoot.pipe( + Effect.retry(abandonCleanupSchedule), Effect.catch((cause) => Effect.logWarning("Abandoned stack could not remove its container-env root", cause), ), ), ), - Effect.andThen( - network.releaseStack.pipe( - Effect.catch((cause) => - Effect.logWarning("Abandoned stack could not release its port reservations", cause), - ), - ), - ), definitionGate.withPermits(1), Effect.withSpan("Owner.abandonNamespace", { attributes: { stack_id: stackId } }), ), + recoverDraining: network.recoverDrain, }, setDraining: (value) => Ref.set(draining, value), getServing: Ref.get(draining).pipe(Effect.map((isDraining) => !isDraining)), diff --git a/packages/stack/src/Proxy.ts b/packages/stack/src/Proxy.ts index 9faaa36513..02b8cb8241 100644 --- a/packages/stack/src/Proxy.ts +++ b/packages/stack/src/Proxy.ts @@ -24,6 +24,8 @@ export interface TcpListener { ) => Effect.Effect; /** Stops accepting new connections on the listener; established connections keep flowing. */ readonly stopAccepting: Effect.Effect; + /** Reverses `stopAccepting` (F6): recovers a listener left refusing after a failed drain. */ + readonly resumeAccepting: Effect.Effect; /** The count of established client connections, observable until it reaches 0. */ readonly outstandingConnections: SubscriptionRef.SubscriptionRef; /** Destroys every established connection immediately. */ @@ -147,6 +149,9 @@ export const bindTcp = ( stopAccepting: Effect.sync(() => { accepting = false; }), + resumeAccepting: Effect.sync(() => { + accepting = true; + }), outstandingConnections, cutAll: Effect.sync(() => { for (const socket of sockets) socket.destroy(); diff --git a/packages/stack/src/Service.integration.test.ts b/packages/stack/src/Service.integration.test.ts index 8f3990038e..50d9082718 100644 --- a/packages/stack/src/Service.integration.test.ts +++ b/packages/stack/src/Service.integration.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "@effect/vitest"; import { Deferred, Effect, Exit, Fiber, Option, Queue, Ref, Scope, Stream } from "effect"; import { makeStandaloneService } from "../tests/standalone-service.ts"; +import { LifecycleEvent } from "./Lifecycle.ts"; import type { Status } from "./Orchestrator.ts"; import { makeService, @@ -633,6 +634,47 @@ describe("service execution", () => { ), ); + it.live( + "reports confirmed termination when removeData cleans up a session a failed stop retained (F1)", + () => + Effect.scoped( + Effect.gen(function* () { + // Direct `makeService`, not `makeStandaloneService`: abandonment calls `core.removeData` + // without ever going through the orchestrator's own dispatch, which already retries and + // reports a failed stop correctly on its own (`Lifecycle.ts`'s `retryCleanup`) and so + // never exercises this path. Only a direct `removeData` call does. + const plans = yield* Queue.unbounded(); + const events = yield* Ref.make>([]); + const service = yield* makeService(makeDefinition(plans), { + id: "database-removedata-retry", + config: { version: 17 }, + report: (event) => Ref.update(events, (current) => [...current, event]), + }); + const plan = yield* makeRuntimePlan; + yield* Queue.offer(plans, plan); + yield* open(plan.launchGate); + yield* open(plan.healthGate); + yield* service.launch(1, { version: 17 }); + + yield* Ref.set(plan.stopFailure, true); + yield* service.stop(1, { operation: "stop", discard: true }); + expect((yield* Ref.get(events)).some((event) => event._tag === "StopFailed")).toBe(true); + + // The retained session's retry, through `removeData` alone this time. + yield* open(plan.stopGate); + yield* open(plan.removeGate); + yield* service.removeData(); + + expect( + (yield* Ref.get(events)).some((event) => event._tag === "Exited"), + "the retry's confirmed termination must reach the reducer, or it stays Stopping forever", + ).toBe(true); + expect((yield* Ref.get(plan.state)).removed).toBe(true); + expect((yield* service.get).registered).toBe(false); + }), + ), + ); + it.live( "keeps the registration and data of a running instance whose destroy fails to stop it", () => diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index 8f1b9131dd..ec4d6a567b 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -490,7 +490,21 @@ export const makeService = ( yield* update({ currentOperation: "destroy" }); yield* Effect.gen(function* () { const leftover = yield* Ref.get(current); - if (leftover !== undefined) yield* stopRecord(leftover, true); + if (leftover !== undefined) { + yield* stopRecord(leftover, true); + // F1: a previous stop attempt may have failed and reported `StopFailed`, leaving the + // reducer in `Stopping` forever; this retry's own confirmed termination must reach it + // too, the same way `stop`'s own successful path does, or the reducer (and anything + // gated on it, such as `NetworkNamespace.release`'s endpoint check) never learns. + yield* report( + LifecycleEvent.Exited({ + id, + generation: leftover.generation, + cause: undefined, + requested: true, + }), + ); + } const dataScope = yield* Scope.fork(owner, "parallel"); yield* definition .removeData({ id, config: yield* Ref.get(config), scope: dataScope }) diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index 17f2ed8d02..e0bdf450d3 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -379,19 +379,16 @@ it.live.skipIf(process.platform === "win32")( (yield* containers(stackId, dataRoot)).length, "the owner runs the mail container", ).toBeGreaterThan(0); + const status = yield* client.status({ id: mail.id }); + const port = status.endpoints.find((endpoint) => endpoint.name === "http")?.port; + if (port === undefined) return yield* Effect.die("Missing mail http endpoint"); - // Deletes every discovery-relevant entry in the stack directory, not just `state.json`: the - // claims journal and storage marker are gone too, so cleanup can only come from the owner's - // in-memory resources, never from on-disk discovery. `owner.log` is spared only because it - // is diagnostic output this test reads on failure, never a file the owner itself discovers - // or recovers from. The lease file is removed, so this relies on budgeted polling rather - // than a lease-release subscription. - for (const entry of yield* fs.readDirectory(`${stateRoot}/${stackId}`)) - if (entry !== "owner.log") - yield* fs.remove(`${stateRoot}/${stackId}/${entry}`, { - recursive: true, - force: true, - }); + // Deletes the actual `` itself (F2), not just the entries inside + // `/`: its sibling `.registry-lock.sqlite` (`Ports.ts`'s per-root + // reservation registry) is gone too, so cleanup can only come from the owner's in-memory + // resources, and must never open that registry at all. The lease file goes with it, so + // this relies on budgeted polling rather than a lease-release subscription. + yield* fs.remove(stateRoot, { recursive: true, force: true }); yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( Effect.retry({ schedule: Schedule.spaced("1 second"), @@ -405,14 +402,44 @@ it.live.skipIf(process.platform === "win32")( yield* containers(stackId, dataRoot), "abandonment removes containers from in-memory resources alone", ).toEqual([]); - expect( - yield* fs.exists(`${stateRoot}/${stackId}/state.json`), - "no registration is republished", - ).toBe(false); - expect( - yield* fs.exists(`${stateRoot}/${stackId}/claims.json`), - "no claims journal is republished", - ).toBe(false); + + // The port reservation is never released by abandonment (simplification over F2/F7): + // a fresh stack can still claim the exact same port, through `Ports.ts`'s own lazy + // reclamation once the former holder's registration is confirmed gone. + const reclaimedBase = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-abandon-docker-root-reclaim-", + }); + const reclaimedStackId = `abandon-root-reclaim-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const reclaimedDataRoot = yield* makeDockerDatabaseRoot( + "stack-abandon-docker-root-reclaim-data-", + reclaimedStackId, + ).pipe(Effect.flatMap(fs.realPath)); + const reclaimedStateRoot = path.dirname(path.dirname(reclaimedDataRoot)); + yield* Effect.addFinalizer(() => + removeContainers(reclaimedStackId, reclaimedDataRoot).pipe(Effect.ignore), + ); + const reclaimedAccess = yield* startHost( + reclaimedStateRoot, + cacheRoot, + reclaimedStackId, + `${reclaimedBase}/project`, + ); + const reclaimedClient = yield* ownerClient(reclaimedAccess); + const reclaimedMail = yield* reclaimedClient.createService({ + service: "mail", + config: {}, + endpoints: { http: { port } }, + }); + yield* reclaimedClient.startService({ id: reclaimedMail.id }); + yield* reclaimedClient.readyService({ id: reclaimedMail.id }); + const reclaimedStatus = yield* reclaimedClient.status({ id: reclaimedMail.id }); + expect(reclaimedStatus.endpoints.find((endpoint) => endpoint.name === "http")?.port).toBe( + port, + ); + yield* shutdownOwner(reclaimedAccess, true); + yield* waitForOwnerExit(reclaimedAccess.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.timeout("15 seconds"), + ); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 180_000 }, diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts index 13bac03c25..daebe04f5c 100644 --- a/packages/stack/src/StackHost.integration.test.ts +++ b/packages/stack/src/StackHost.integration.test.ts @@ -314,6 +314,62 @@ it.live("keeps serving when namespace shutdown fails", () => ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); +it.live("recovers network admission after a shutdown that fails once drain has begun (F6)", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-host-drain-recovery-" }); + const state = yield* stateFor(`${root}/state`); + const saved = { + id: "stack", + runtime: "native" as const, + identity: { projectRoot: root, branchContext: "main", stackName: "host-drain-recovery" }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + const owner = yield* ownerFor({ + saved, + state, + root: `${root}/data`, + cacheRoot: "/tmp/supabase-stack-artifacts", + }); + const failedOwner = { + ...owner, + namespace: { + ...owner.namespace, + // Lets the real drain run (so `Network`'s own internal draining flag is genuinely set), + // then fails afterward: reproduces "shutdown fails after drain begins" without reaching + // into `Network`'s internals directly. + stop: owner.namespace.stop.pipe( + Effect.andThen( + Effect.fail(new OrchestratorError({ operation: "stop", message: "cleanup failed" })), + ), + ), + }, + }; + const { runtime } = yield* inProcessRuntime(failedOwner, state, root); + const client = yield* ownerClient(runtime.access); + const failure = yield* shutdownOwner(runtime.access, false).pipe(Effect.flip); + expect(failure.message).toContain("cleanup failed"); + + // F6: a drain that already began during the failed shutdown must not block ordinary + // service recovery forever; starting a service again must bind its listener, not refuse it + // as still draining. + const mail = yield* client.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* client.startService({ id: mail.id }); + yield* client.readyService({ id: mail.id }); + yield* shutdownOwner(runtime.access, true); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + it.live("reports destroy and fallback stop failures together", () => Effect.scoped( Effect.gen(function* () { @@ -1128,24 +1184,6 @@ const httpRoundTrip = (port: number, agent: Http.Agent) => }, ); -/** - * Connects a raw socket to `port` without speaking HTTP, so Node's own keep-alive idle timeout - * (5 s by default, shorter than the 10 s drain deadline) never applies to it. The listener proxies - * whatever bytes flow without needing a real protocol, so this still counts as one established - * connection for drain purposes. - */ -const openRawSocket = (port: number) => - Effect.acquireRelease( - Effect.callback((resume) => { - const socket = Net.connect(port, "127.0.0.1"); - socket.on("error", () => {}); - socket.once("connect", () => resume(Effect.succeed(socket))); - socket.once("error", (cause) => resume(Effect.fail(hostTestError(cause)))); - return Effect.void; - }), - (socket) => Effect.sync(() => socket.destroy()), - ); - /** `true` once a fresh one-shot request to `port` is refused (reset) rather than answered. */ const isHttpRefused = (port: number) => Effect.acquireRelease( @@ -1184,10 +1222,9 @@ it.live( (agent) => Effect.sync(() => agent.destroy()), ); const first = yield* httpRoundTrip(port, agent); - expect( - Number.isFinite(first.status), - "the first request gets a real response before drain", - ).toBe(true); + // Exactly 200, not merely finite: a proxy-generated 502 (the backend unreachable) is + // just as finite a status and must not pass as "a real response". + expect(first.status, "the first request gets a real response before drain").toBe(200); const shutdown = yield* Effect.forkScoped(shutdownOwner(runtime.access, false)); yield* waitUntilRefused(port); expect(first.socket.destroyed, "the established connection survives the refusal").toBe( @@ -1199,9 +1236,9 @@ it.live( "the second request reuses the same already-established connection", ).toBe(first.socket); expect( - Number.isFinite(second.status), + second.status, "a request on the established connection still completes during drain", - ).toBe(true); + ).toBe(200); // The client finishes on its own, well before the drain deadline. agent.destroy(); yield* Fiber.join(shutdown).pipe( @@ -1219,33 +1256,40 @@ it.live( ); // Drives the deadline with a shortened `Network.ShutdownDrainDeadline` (the internal reference -// production always defaults to 10 s) rather than asserting a wall-clock lower bound: a timed -// assertion only proves this test's own timer started before drain's, which a loaded CI runner can -// easily violate without the drain logic itself being wrong. Ordering is observed instead: the -// connection is confirmed still open while draining, then confirmed cut once stop completes. +// production always defaults to 10 s) rather than `TestClock`: this in-process owner's other +// timers (lease, idle, cooldown, readiness polling) share the same `Clock` through one layer +// composition, so virtualizing it for the deadline alone risks silently changing those unrelated +// subsystems too. The shortened real deadline keeps this test's own timing assertion-free in turn: +// ordering is observed (still open mid-drain, then cut once stop completes), never timed. it.live( "cuts a hanging connection at the drain deadline, then stop completes", () => Effect.scoped( Effect.gen(function* () { const { runtime, port } = yield* withRunningMail("stack-host-drain-deadline-"); - // Proves the listener is genuinely functional before testing the hang. A one-shot agent, not - // the hanging connection itself: Node's own ~5 s keep-alive idle timeout would otherwise cut - // an idle HTTP connection before even a shortened drain deadline has a chance to. - const proof = yield* Effect.acquireUseRelease( - Effect.sync(() => new Http.Agent({ keepAlive: false })), - (agent) => httpRoundTrip(port, agent), + const agent = yield* Effect.acquireRelease( + Effect.sync(() => new Http.Agent({ keepAlive: true, maxSockets: 1 })), (agent) => Effect.sync(() => agent.destroy()), ); - expect(Number.isFinite(proof.status), "the listener answers before the hang").toBe(true); - const socket = yield* openRawSocket(port); + // Establishes progress on the exact socket under test, not a different one: the listener + // has genuinely accepted and answered on it before drain, closing the race a bare + // client-side `connect` would otherwise leave against the server's own acceptance. + const first = yield* httpRoundTrip(port, agent); + expect(first.status, "the listener answers before the hang").toBe(200); const shutdown = yield* Effect.forkScoped(shutdownOwner(runtime.access, false)); - // Observed, not timed: confirms drain has started (new connections already refused) while - // the hanging connection is still deliberately held open, ahead of the deadline cut. yield* waitUntilRefused(port); - expect(socket.destroyed, "the established connection survives into the drain window").toBe( - false, - ); + // A second, real in-flight request on that same connection, started only once drain is + // already refusing new ones, proves it keeps flowing mid-drain rather than merely sitting + // idle and unnoticed. + const second = yield* httpRoundTrip(port, agent); + expect(second.socket, "reuses the same established connection").toBe(first.socket); + expect(second.status, "a request on it still completes during drain").toBe(200); + expect( + second.socket.destroyed, + "the established connection survives into the drain window", + ).toBe(false); + // Deliberately left idle (no further request, the agent never closed) past this point, so + // the deadline cut — not a voluntary close — is what ends it. yield* Fiber.join(shutdown).pipe( Effect.timeoutOrElse({ duration: "10 seconds", @@ -1255,7 +1299,7 @@ it.live( ), }), ); - expect(socket.destroyed, "the hanging connection is cut at the deadline").toBe(true); + expect(second.socket.destroyed, "the hanging connection is cut at the deadline").toBe(true); }), ).pipe( Effect.provideService(Network.ShutdownDrainDeadline, "2 seconds"), diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index 5463ac2e4a..e0365a2830 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -325,7 +325,11 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( ? Effect.failCause(cause) : owner .setDraining(false) + // F6: drain may already have run (and so left the network's own + // admission closed) before this later stage failed; recovering it here + // keeps a start/restart that follows from refusing as still draining. .pipe( + Effect.andThen(owner.namespace.recoverDraining), Effect.andThen(gate.withPermits(1)(Ref.set(current, undefined))), Effect.andThen(Effect.failCause(cause)), ), @@ -350,6 +354,14 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( Effect.catchCause((cause) => Effect.logError(`Abandoned stack: ${label} failed`, cause)), ); const abandonResources = settled("abandon namespace cleanup", owner.namespace.abandon); + // F5: a retained native command's termination can still be failing when this runs; retried + // under the same policy as every other abandonment cleanup stage (confirmed, or a confirmed + // permanently unreachable engine) before the owner is permitted to exit. Namespace cleanup + // below still runs on its own regardless of how long this takes. + const retainedCommandCleanup = settled( + "command runner cleanup", + runner.cleanup.pipe(Effect.retry(Owner.abandonCleanupSchedule)), + ); const abandon = Effect.fn("StackHost.abandon")(() => gate .withPermits(1)( @@ -361,7 +373,7 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( const cleanup = restore( Effect.gen(function* () { yield* settled("stopping attachments", attachments.stopAll); - yield* settled("command runner cleanup", runner.cleanup); + yield* retainedCommandCleanup; yield* abandonResources; }), ).pipe( @@ -379,14 +391,22 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( .pipe( Effect.flatMap((record) => Fiber.join(record.fiber).pipe( + Effect.exit, // `record.fiber` may be an in-flight normal shutdown, typed to report `StackError`; // abandonment never fails the caller (the registration poll), so it is logged. - Effect.catch((cause) => Effect.logError("Stack shutdown failed", cause)), - Effect.andThen( - // A joined plain stop deliberately preserves storage and port reservations; F6 - // cleanup still has to run. A joined destroy already confirmed both, so running - // it again would be redundant (and safe: the shared cleanup path is idempotent). - record.destroy ? Effect.void : abandonResources, + Effect.tap((joined) => + Exit.isFailure(joined) + ? Effect.logError("Stack shutdown failed", joined.cause) + : Effect.void, + ), + Effect.flatMap((joined) => + // F4: a joined destroy only confirmed both data removal and reservation release + // when it actually succeeded; a failed destroy can have fallen back to a plain + // stop (which preserves both) or failed before removing anything, so either way + // F6 cleanup still has to run. Inspecting the outcome, not the requested mode, + // is what tells the two apart; running it again on an already-confirmed destroy + // is redundant but safe, since the shared cleanup path is idempotent. + record.destroy && Exit.isSuccess(joined) ? Effect.void : abandonResources, ), ), ), diff --git a/packages/stack/src/composition/Supabase.unit.test.ts b/packages/stack/src/composition/Supabase.unit.test.ts index b5ebab47a0..6fb5e2a36e 100644 --- a/packages/stack/src/composition/Supabase.unit.test.ts +++ b/packages/stack/src/composition/Supabase.unit.test.ts @@ -42,6 +42,7 @@ const makeInstance = ( bind: Effect.void, close: Effect.void, confirmRemoved: Effect.void, + release: Effect.void, hasEndpoint: options.hasEndpoint ?? true, inputs: options.inputs ?? [], outputs: options.outputs ?? {}, diff --git a/packages/stack/src/runtime/Container.integration.test.ts b/packages/stack/src/runtime/Container.integration.test.ts index 9e16d07d1e..9d3d0bc3ca 100644 --- a/packages/stack/src/runtime/Container.integration.test.ts +++ b/packages/stack/src/runtime/Container.integration.test.ts @@ -27,6 +27,7 @@ import { HttpClient } from "effect/unstable/http"; import { ContainerLaunchError, DOCKER_HOST_ALIAS, + engineUnreachable, makeContainerRuntime, makeHostGateway, resolveEngineTarget, @@ -1493,6 +1494,63 @@ describe("container process adapter", () => { ), ); + it.live( + "classifies a real connection refusal to the engine as permanently unavailable (F3)", + () => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + // A real connection attempt, not a mocked one: nothing listens on this loopback port, so + // the docker CLI's own refusal is what `errorFor` classifies, at its one true source. + const failure = yield* resolveEngineTarget(spawner).pipe(Effect.flip); + expect( + engineUnreachable(failure), + "a real daemon connection refusal classifies as engine-unavailable", + ).toBe(true); + }).pipe( + Effect.provide( + Layer.merge( + NodeServices.layer, + ConfigProvider.layer( + ConfigProvider.fromEnvRecord({ DOCKER_HOST: "tcp://127.0.0.1:1" }), + ), + ), + ), + ), + ); + + it.live("classifies a real missing engine binary as permanently unavailable (F3)", () => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fs = yield* FileSystem.FileSystem; + // A directory deliberately containing no `docker` binary, real `PATH` lookup included: the + // engine's own spawn failure, not a stubbed `ChildProcessSpawner`, is what `errorFor` + // classifies. + const emptyBinDir = yield* fs.makeTempDirectoryScoped({ prefix: "engine-missing-bin-" }); + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the spawned child inherits PATH; this is not application config. + const originalPath = process.env.PATH; + yield* Effect.acquireRelease( + Effect.sync(() => { + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. + process.env.PATH = emptyBinDir; + }), + () => + Effect.sync(() => { + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. + process.env.PATH = originalPath; + }), + ); + const failure = yield* resolveEngineTarget(spawner).pipe(Effect.flip); + expect( + engineUnreachable(failure), + "a real spawn ENOENT for the missing docker binary classifies as engine-unavailable", + ).toBe(true); + }).pipe( + Effect.provide( + Layer.merge(NodeServices.layer, ConfigProvider.layer(ConfigProvider.fromEnvRecord({}))), + ), + ), + ); + it.live( "keeps helper creation, runtime launches, password reconcile and reconcile pinned to the context an owner resolved at startup, despite a later context switch", () => diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index 91b759e4e4..9d28c360f4 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -93,12 +93,44 @@ export interface ContainerRuntime { ) => Effect.Effect; } -const errorFor = (operation: string, cause: unknown) => - new ContainerError({ +/** A spawn failure for a missing engine CLI binary; it carries no message pattern to test. */ +const isMissingEngineBinary = (cause: unknown): boolean => + cause instanceof PlatformError.PlatformError && + cause.reason._tag === "NotFound" && + cause.reason.method === "spawn"; + +/** + * Matches an engine CLI that is missing or reports a daemon that is not listening, not one that + * rejects the caller. Windows' `error during connect` also wraps authentication and TLS failures, + * so only its refused or missing-endpoint causes match. Matches regardless of which wrapping error + * type carried the message (a bare `ContainerError`, or another operation's error whose message + * preserves the underlying CLI failure text, such as a database storage cleanup failure). + */ +const engineUnreachableMessage = (message: string): boolean => + /cannot connect to the docker daemon|connection refused|connect: no such file or directory|error during connect:[^\n]*(?:docker daemon is not running|the system cannot find the file specified)/iu.test( + message, + ); + +/** + * Whether `cause` (a spawn failure for a missing engine binary) or `message` (the engine CLI's + * own unreachable-daemon phrasing) signals a permanently unavailable engine (F3). Classified once, + * here, at every engine invocation boundary (`errorFor`, below), and carried onward through + * wrapping as the error's own `reason` rather than re-matched; a caller that only ever sees a + * message (database storage, which invokes the engine CLI directly rather than through this + * module's `ContainerError`) calls this too, at its own equivalent source. + */ +export const isEngineUnavailable = (cause: unknown, message: string): boolean => + isMissingEngineBinary(cause) || engineUnreachableMessage(message); + +const errorFor = (operation: string, cause: unknown): ContainerError => { + const message = cause instanceof Error ? cause.message : String(cause); + return new ContainerError({ operation, - message: cause instanceof Error ? cause.message : String(cause), + message, cause, + ...(isEngineUnavailable(cause, message) ? { reason: "engine-unavailable" as const } : {}), }); +}; /** Labels containers this run creates, when `SUPABASE_STACK_TEST_RUN` is set. */ const testRunLabelArgs = readTestRunLabelArgs.pipe( @@ -118,24 +150,8 @@ const transientPullFailure = (error: ContainerError) => error.message, ); -/** - * Matches an engine CLI that is missing or reports a daemon that is not listening, not one that - * rejects the caller. Windows' `error during connect` also wraps authentication and TLS failures, - * so only its refused or missing-endpoint causes match. Matches regardless of which wrapping error - * type carried the message (a bare `ContainerError`, or another operation's error whose message - * preserves the underlying CLI failure text, such as a database storage cleanup failure). - */ -export const engineUnreachableMessage = (message: string): boolean => - /cannot connect to the docker daemon|connection refused|connect: no such file or directory|error during connect:[^\n]*(?:docker daemon is not running|the system cannot find the file specified)/iu.test( - message, - ); - -/** Also matches a missing engine CLI binary, which never produces a message to test. */ -export const engineUnreachable = (error: ContainerError) => - (error.cause instanceof PlatformError.PlatformError && - error.cause.reason._tag === "NotFound" && - error.cause.reason.method === "spawn") || - engineUnreachableMessage(error.message); +/** Classified once, at construction (`errorFor`), and simply read back here. */ +export const engineUnreachable = (error: ContainerError) => error.reason === "engine-unavailable"; /** Runs one engine CLI invocation outside any pinned target, for resolving that target itself. */ const runRaw = ( @@ -645,13 +661,14 @@ export const makeContainerRuntime = (options: { // Not `makeTempDirectoryScoped`: its scoped cleanup removes without `force`, then dies on // any failure, including a confirmed ENOENT when this stack's whole root (this directory's // owned ancestor) is already gone — turning an ordinary stop into an unrecoverable defect - // (F5/F1 root cause). A plain temp directory plus a tolerant finalizer treats "already gone" - // as already cleaned up, matching this directory's own owned-and-disposable nature. - const directory = yield* fs - .makeTempDirectory({ directory: containerEnvRoot, prefix: "container-" }) - .pipe(Effect.mapError((cause) => errorFor("environment", cause))); - yield* Effect.addFinalizer(() => - fs.remove(directory, { recursive: true, force: true }).pipe(Effect.ignore), + // (F5/F1 root cause). `acquireRelease` keeps the plain `makeTempDirectory` and installing its + // tolerant, `force: true` finalizer uninterruptible together (F8), so an interruption between + // the two can never leave the directory created but unregistered for cleanup. + const directory = yield* Effect.acquireRelease( + fs + .makeTempDirectory({ directory: containerEnvRoot, prefix: "container-" }) + .pipe(Effect.mapError((cause) => errorFor("environment", cause))), + (value) => fs.remove(value, { recursive: true, force: true }).pipe(Effect.ignore), ); const envPath = path.join(directory, "environment"); yield* fs @@ -754,6 +771,7 @@ export const makeContainerRuntime = (options: { operation: error.operation, message: `${error.message} (container name ${name})`, cause: error, + ...(error.reason === undefined ? {} : { reason: error.reason }), }), ), Effect.catchTag("TimeoutError", () => @@ -1056,17 +1074,10 @@ export const removeContainerById = Effect.fn("Container.removeContainerById")( }), ).pipe( Effect.timeout("30 seconds"), - Effect.mapError((cause) => { - const error = cause instanceof ContainerError ? cause : errorFor("cleanup", cause); - return engineUnreachable(error) - ? new ContainerError({ - operation: error.operation, - message: error.message, - cause: error.cause, - reason: "engine-unavailable", - }) - : error; - }), + // `errorFor` already classifies at construction; nothing further to reclassify here. + Effect.mapError((cause) => + cause instanceof ContainerError ? cause : errorFor("cleanup", cause), + ), ); }), ); diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index a9fb7fc87c..e05c576e0f 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -19,7 +19,7 @@ import { testRunLabelArgs as readTestRunLabelArgs } from "../internal/test-run-l import type * as Claims from "../namespace/Claims.ts"; import * as Publication from "../namespace/Publication.ts"; import { - engineUnreachableMessage, + isEngineUnavailable, type ContainerRuntime, type EngineTarget, } from "../runtime/Container.ts"; @@ -100,9 +100,10 @@ export interface DockerDatabaseStorage { } // Snapshot failures keep the protocol's operation so both engines report the same step. -// The engine CLI's own stderr text is the only unreachable-daemon signal this module ever sees -// (it invokes the raw CLI directly, not through `runtime/Container.ts`'s typed `ContainerError`), -// so the message is classified here, once, at the source, and carried onward as `reason`. +// A spawn failure for a missing engine CLI binary, or the engine CLI's own stderr text, is the +// only unreachable-daemon signal this module ever sees (it invokes the raw CLI directly, not +// through `runtime/Container.ts`'s typed `ContainerError`), so both are classified here, once, at +// the source (F3), and carried onward as `reason`. const errorFor = (operation: string, cause: unknown) => Schema.is(DockerDatabaseStorageError)(cause) ? cause @@ -111,13 +112,13 @@ const errorFor = (operation: string, cause: unknown) => operation: cause.operation, message: cause.message, cause, - ...(engineUnreachableMessage(cause.message) ? { reason: "engine-unavailable" } : {}), + ...(isEngineUnavailable(cause, cause.message) ? { reason: "engine-unavailable" } : {}), }) : new DockerDatabaseStorageError({ operation, message: failureMessage(cause), cause, - ...(engineUnreachableMessage(failureMessage(cause)) + ...(isEngineUnavailable(cause, failureMessage(cause)) ? { reason: "engine-unavailable" } : {}), }); diff --git a/packages/stack/tests/standalone-service.ts b/packages/stack/tests/standalone-service.ts index 39d8ff6640..031a7233e0 100644 --- a/packages/stack/tests/standalone-service.ts +++ b/packages/stack/tests/standalone-service.ts @@ -35,6 +35,7 @@ export const makeStandaloneService = Effect.fn("Test.makeStandaloneService")(fun bind: Effect.void, close: Effect.void, confirmRemoved: Effect.void, + release: Effect.void, hasEndpoint: true, inputs: [], outputs: {}, From 7bb1feb718dffb83f2518923b1459c4d60994cea Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 09:54:26 +0200 Subject: [PATCH 037/142] refactor(stack): unify shutdown modes and probe engine availability for abandonment Collapses abandonment into a mode of the existing shutdown pipeline (stop | destroy | abandon) claimed through the same gate and current record, so a shutdown already in flight is joined rather than raced. Replaces engine-unavailable message/reason classification with a direct probe of the pinned engine target, and adds a registration-independent sweep that removes every container carrying the stack's identity label through the pinned engine, covering leaked helpers the per-instance cleanup misses. Makes the shutdown drain deadline an injectable event so tests can gate it deterministically instead of racing a timer. --- packages/stack/src/Network.ts | 15 +- packages/stack/src/Owner.ts | 117 +++--- ...ost.container-shutdown.integration.test.ts | 78 ++++ .../stack/src/StackHost.integration.test.ts | 281 ++++++++++++--- packages/stack/src/StackHost.ts | 339 ++++++++++-------- .../src/runtime/Container.integration.test.ts | 91 +++-- packages/stack/src/runtime/Container.ts | 109 +++--- .../src/storage/DockerDatabaseStorage.ts | 22 +- 8 files changed, 695 insertions(+), 357 deletions(-) diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index 8b44a93951..21e3712e02 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -1,7 +1,6 @@ import { Context, Data, - type Duration, Effect, Exit, Layer, @@ -21,13 +20,15 @@ import { makeHttpProxy, type HttpProxy, type HttpRoute } from "./HttpProxy.ts"; export type NetworkRuntime = "native" | "docker"; /** - * The shutdown-drain deadline (F5): established connections keep flowing until this elapses. - * Internal only, following the `RegistrationCheckInterval` pattern: tests shorten it through this - * reference; production never reads it from an environment variable or `Config`. + * The shutdown-drain deadline (F5): established connections keep flowing until this event + * completes. An injectable event rather than a duration (pass 3, F7), so a test can gate exactly + * when it fires through a `Deferred` instead of racing a shortened real timer; production always + * defaults to a plain 10-second sleep. Internal only, following the `RegistrationCheckInterval` + * pattern: production never reads it from an environment variable or `Config`. */ -export const ShutdownDrainDeadline = Context.Reference( +export const ShutdownDrainDeadline = Context.Reference>( "@supabase/stack/ShutdownDrainDeadline", - { defaultValue: () => "10 seconds" }, + { defaultValue: () => Effect.sleep("10 seconds") }, ); /** The accept/outstanding/cut-all surface every listener kind (`TcpListener`, `HttpProxy`) shares. */ @@ -453,7 +454,7 @@ const makeNetwork = (options: { discard: true, }).pipe( Effect.as("quiesced" as const), - Effect.race(Effect.sleep(deadline).pipe(Effect.as("deadline" as const))), + Effect.race(deadline.pipe(Effect.as("deadline" as const))), ); if (outcome === "deadline") yield* Effect.forEach(handles, (handle) => handle.cutAll, { discard: true }); diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index f73bf4f751..deabcefcec 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -1,15 +1,16 @@ import { + Cause, Context, Crypto, Duration, Effect, + Exit, Fiber, FileSystem, Layer, Option, Path, Ref, - Schedule, Schema, Scope, Semaphore, @@ -65,7 +66,6 @@ import { } from "./services/Catalog.ts"; import type { CatalogError } from "./services/Recipe.ts"; import * as Container from "./runtime/Container.ts"; -import { DockerDatabaseStorageError } from "./storage/DockerDatabaseStorage.ts"; import * as Claims from "./namespace/Claims.ts"; import { namespaceError } from "./namespace/Capabilities.ts"; import { lstatPath } from "./namespace/drivers/FileSystem.ts"; @@ -196,6 +196,8 @@ export interface Interface { */ readonly recoverDraining: Effect.Effect; }; + /** Direct probe of the pinned engine target; always `true` for a native stack (pass 3, B). */ + readonly engineAvailable: Effect.Effect; readonly setDraining: (draining: boolean) => Effect.Effect; readonly getServing: Effect.Effect; } @@ -230,49 +232,38 @@ const rpcError = (operation: string) => Effect.mapError((cause: unknown) => stackError(operation, cause)); /** - * A container engine confirmed unreachable (F5): the shared cleanup path reports this as a plain - * `ServiceError` wrapping whatever the failing operation raised (a `ContainerError` from workload - * removal, or a `DockerDatabaseStorageError` from database volume cleanup, among others), so the - * daemon's own unreachable-daemon phrasing in the message is the one signal common to all of them. + * Retries `operation` with a bounded, doubling backoff until it succeeds or `engineAvailable` + * resolves `false` (pass 3, B): engine availability is a direct probe, never a message or cause + * classification, so this is the one place every abandonment cleanup stage decides whether a + * failure is worth retrying. A `false` probe fails with the operation's own last error; an + * interruption or defect is never retried. */ -/** - * A container engine confirmed unreachable (F5), checked as a typed reason through the cause - * chain rather than by matching message text: a direct `ContainerError` from workload removal, or - * a `DockerDatabaseStorageError` from database volume cleanup, whose own `reason` field is set - * once, at the raw engine CLI invocation that is the only place either module can observe it. - */ -const engineUnreachableCleanup = (error: ServiceError) => - (error.cause instanceof Container.ContainerError && - error.cause.reason === "engine-unavailable") || - (Schema.is(DockerDatabaseStorageError)(error.cause) && - error.cause.reason === "engine-unavailable"); - -/** - * Backoff for abandonment cleanup retries: bounded delay, unbounded attempts. Per F6's final - * decision, cleanup retries until it is confirmed or the engine is confirmed permanently - * unreachable — never until a fixed time budget runs out while the engine might still recover. - * Exported so every abandonment cleanup stage shares one retry policy, including `StackHost.ts`'s - * retained native-command cleanup, which is outside this module's own cleanup paths. - */ -export const abandonCleanupSchedule = Schedule.exponential("200 millis", 2).pipe( - Schedule.modifyDelay(({ duration }) => - Effect.succeed(Duration.min(duration, Duration.seconds(5))), - ), -); +export const retryUntilConfirmed = ( + operation: Effect.Effect, + engineAvailable: Effect.Effect, +): Effect.Effect => + Effect.gen(function* () { + let delay = Duration.millis(200); + while (true) { + const result = yield* Effect.exit(operation); + if (Exit.isSuccess(result)) return result.value; + const failure = Cause.findErrorOption(result.cause); + if (Option.isNone(failure)) return yield* Effect.failCause(result.cause); + if (!(yield* engineAvailable)) return yield* Effect.fail(failure.value); + yield* Effect.sleep(delay); + delay = Duration.min(Duration.times(delay, 2), Duration.seconds(5)); + } + }); /** * Cleans up one abandoned instance through the same confirmed, serialized execution path destroy * uses (`core.removeData`, under the service's execution lock, retrying only steps a previous * attempt didn't finish), but without a `confirm` step: abandonment never touches the registration. * Retries transient failures; only a confirmed-unreachable engine is logged and left behind, so a - * recoverable failure never blocks the owner's exit (F1, F5). + * recoverable failure never blocks the owner's exit. */ -const cleanupAbandonedInstance = (entry: Entry) => - entry.core.removeData().pipe( - Effect.retry({ - schedule: abandonCleanupSchedule, - while: (error) => !engineUnreachableCleanup(error), - }), +const cleanupAbandonedInstance = (entry: Entry, engineAvailable: Effect.Effect) => + retryUntilConfirmed(entry.core.removeData(), engineAvailable).pipe( Effect.catch((error) => Effect.logWarning( `Abandoned stack could not confirm cleanup of ${entry.service} ${entry.id}; leaving it behind`, @@ -322,8 +313,15 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { const crypto = Context.get(services, Crypto.Crypto); const path = Context.get(services, Path.Path); const fs = Context.get(services, FileSystem.FileSystem); + const spawner = Context.get(services, ChildProcessSpawner.ChildProcessSpawner); const network = yield* Network.Service; const portReservations = yield* PortReservations.Service; + // A native stack has no engine to lose, so every cleanup failure is retried forever; a docker + // stack probes the pinned target directly (pass 3, B), never by classifying a failure's cause. + const engineAvailable: Effect.Effect = + options.engineTarget === undefined + ? Effect.succeed(true) + : Container.probeEngineReachable(spawner, options.engineTarget); // Shared per-launch environment-file scratch directory (Container.ts), owned by this stack as a // whole rather than any one instance; registration-independent so destroy and abandonment (F7) // both reach it without depending on `Registry.remove`, which never runs during abandonment. @@ -935,22 +933,54 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { Effect.andThen( orchestrator.instances.pipe( Effect.flatMap((entries) => - Effect.forEach(entries, cleanupAbandonedInstance, { + Effect.forEach(entries, (entry) => cleanupAbandonedInstance(entry, engineAvailable), { concurrency: "unbounded", discard: true, }), ), ), ), - // F7: retried until confirmed, like every other abandonment cleanup stage; its own - // `force: true` only tolerates an already-gone directory, never a transient failure. - // Port reservations are never released here (simplification over F2/F7): a stale + // Registration-independent confirming sweep (pass 3, C): every container carrying this + // stack's identity label, found through the pinned engine directly rather than through + // any in-memory helper registry's own bookkeeping — covers a shared storage helper the + // per-instance cleanup above never removes on its own. + Effect.andThen( + options.engineTarget === undefined + ? Effect.void + : retryUntilConfirmed( + Container.removeStackContainers({ + target: options.engineTarget, + stackId, + }).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.mapError(serviceError("cleanup")), + Effect.filterOrFail( + (remaining) => remaining.length === 0, + (remaining) => + new ServiceError({ + operation: "cleanup", + message: `Containers remain: ${remaining.join(", ")}`, + }), + ), + ), + engineAvailable, + ).pipe( + Effect.catch((cause) => + Effect.logWarning( + "Abandoned stack could not confirm every container removed; leaving it behind", + cause, + ), + ), + ), + ), + // Filesystem-only, so always retried to confirmation regardless of engine availability + // (F7): its own `force: true` tolerates an already-gone directory, never a transient + // failure. Port reservations are never released here (simplification over F2): a stale // reservation stays for `Ports.ts`'s lazy reclamation once the holder's registration is // confirmed gone — exactly the abandonment trigger — so abandonment never touches // `Ports.ts`'s registry at all, even once the state root itself no longer exists. Effect.andThen( - removeContainerEnvRoot.pipe( - Effect.retry(abandonCleanupSchedule), + retryUntilConfirmed(removeContainerEnvRoot, Effect.succeed(true)).pipe( Effect.catch((cause) => Effect.logWarning("Abandoned stack could not remove its container-env root", cause), ), @@ -961,6 +991,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ), recoverDraining: network.recoverDrain, }, + engineAvailable, setDraining: (value) => Ref.set(draining, value), getServing: Ref.get(draining).pipe(Effect.map((isDraining) => !isDraining)), } satisfies Interface; diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index e0bdf450d3..00509807b4 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -326,6 +326,84 @@ it.live.skipIf(process.platform === "win32")( { timeout: 180_000 }, ); +it.live.skipIf(process.platform === "win32")( + "removes an orphaned container by label alone, independent of any instance's own cleanup (pass 3, C)", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-abandon-docker-label-" }); + const stackId = `abandon-label-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const dataRoot = yield* makeDockerDatabaseRoot( + "stack-abandon-docker-label-data-", + stackId, + ).pipe(Effect.flatMap(fs.realPath)); + const stateRoot = path.dirname(path.dirname(dataRoot)); + const cacheRoot = `${base}/cache`; + const state = yield* stateFor(stateRoot); + yield* state.save({ + id: stackId, + runtime: "docker", + identity: { + projectRoot: `${base}/project`, + branchContext: "abandon-docker-label-test", + stackName: stackId, + }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + ports: [], + }); + yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); + const access = yield* launchHost(state, { + stateRoot, + cacheRoot, + stackId, + entrypoint: shortRegistrationPollFixture, + }); + // No registered service at all, so the per-instance cleanup loop has nothing to do: this + // container carries only the stack's identity label, simulating a leaked storage helper + // no claim or helper-registry bookkeeping ever reaches (F5/pass 3, C). Only the + // registration-independent label sweep can remove it. + yield* docker([ + "run", + "-d", + "--name", + `supabase-orphan-${stackId}`, + "--label", + `com.supabase.stack=${stackId}`, + "--label", + `com.supabase.stack-root=${dataRoot}`, + "--label", + "com.supabase.stack-managed=true", + helperImage, + "/bin/sh", + "-c", + "trap : TERM INT; while :; do sleep 3600; done", + ]); + expect( + (yield* containers(stackId, dataRoot)).length, + "the orphan container exists before abandonment", + ).toBeGreaterThan(0); + + const leaseReleased = yield* watchLeaseRelease(stateRoot, stackId); + yield* fs.remove(`${stateRoot}/${stackId}/state.json`); + yield* leaseReleased; + yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.timeout("30 seconds"), + ); + + expect( + yield* containers(stackId, dataRoot), + "abandonment removes the orphan container by label alone", + ).toEqual([]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 180_000 }, +); + it.live.skipIf(process.platform === "win32")( "removes its containers and exits when its whole state root is confirmed gone", () => diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts index daebe04f5c..36165a3bc6 100644 --- a/packages/stack/src/StackHost.integration.test.ts +++ b/packages/stack/src/StackHost.integration.test.ts @@ -370,6 +370,74 @@ it.live("recovers network admission after a shutdown that fails once drain has b ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); +it.live( + "freezes admission and confirms cleanup when abandonment claims ownership after a failing stop settles (pass 3, A)", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-host-abandon-after-stop-", + }); + const state = yield* stateFor(`${root}/state`); + const saved = { + id: "stack", + runtime: "native" as const, + identity: { + projectRoot: root, + branchContext: "main", + stackName: "host-abandon-after-stop", + }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + const owner = yield* ownerFor({ + saved, + state, + root: `${root}/data`, + cacheRoot: "/tmp/supabase-stack-artifacts", + }); + const stopGate = yield* Deferred.make(); + const abandonEntered = yield* Deferred.make(); + const abandonGate = yield* Deferred.make(); + const failedOwner = { + ...owner, + namespace: { + ...owner.namespace, + stop: Deferred.await(stopGate).pipe( + Effect.andThen( + Effect.fail(new OrchestratorError({ operation: "stop", message: "stop failed" })), + ), + ), + abandon: Deferred.succeed(abandonEntered, undefined).pipe( + Effect.andThen(Deferred.await(abandonGate)), + Effect.andThen(owner.namespace.abandon), + ), + }, + }; + const { runtime } = yield* inProcessRuntime(failedOwner, state, root); + const stopFiber = yield* Effect.forkScoped(runtime.shutdown(false)); + // Abandon is requested while the stop is still in flight, never after it has settled: + // the one shutdown pipeline (pass 3, A) waits for it, instead of racing its own cleanup. + const abandonFiber = yield* Effect.forkScoped(runtime.abandon); + yield* Deferred.succeed(stopGate, undefined); + expect(Exit.isFailure(yield* Fiber.join(stopFiber).pipe(Effect.exit))).toBe(true); + yield* Deferred.await(abandonEntered).pipe(Effect.timeout("2 seconds")); + // Admission stays frozen through abandonment's own cleanup, never left reopened by the + // failing stop's own recovery that preceded it (F2). + expect(yield* owner.getServing, "admission stays frozen while abandonment cleans up").toBe( + false, + ); + yield* Deferred.succeed(abandonGate, undefined); + yield* Fiber.join(abandonFiber).pipe(Effect.timeout("2 seconds")); + yield* Deferred.await(runtime.exit).pipe(Effect.timeout("2 seconds")); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + it.live("reports destroy and fallback stop failures together", () => Effect.scoped( Effect.gen(function* () { @@ -1161,28 +1229,32 @@ const withRunningMail = Effect.fn("Test.withRunningMail")(function* (prefix: str * socket that carried it, so a caller can later confirm that same connection's fate. */ const httpRoundTrip = (port: number, agent: Http.Agent) => - Effect.callback<{ readonly status: number; readonly socket: Net.Socket }, HostTestError>( - (resume) => { - const request = Http.request( - { host: "127.0.0.1", port, path: "/", method: "GET", agent }, - (response) => { - // Captured immediately: the agent may detach `response.socket` once the response ends. - const socket = response.socket; - response.resume(); - response.once("end", () => { - if (socket === null) { - resume(Effect.fail(new HostTestError({ message: "Response has no socket" }))); - return; - } - resume(Effect.succeed({ status: response.statusCode ?? 0, socket })); - }); - }, - ); - request.once("error", (cause) => resume(Effect.fail(hostTestError(cause)))); - request.end(); - return Effect.sync(() => request.destroy()); - }, - ); + Effect.callback< + { readonly status: number; readonly socket: Net.Socket; readonly body: string }, + HostTestError + >((resume) => { + const request = Http.request( + { host: "127.0.0.1", port, path: "/", method: "GET", agent }, + (response) => { + // Captured immediately: the agent may detach `response.socket` once the response ends. + const socket = response.socket; + let body = ""; + response.on("data", (chunk: Buffer) => { + body += chunk.toString(); + }); + response.once("end", () => { + if (socket === null) { + resume(Effect.fail(new HostTestError({ message: "Response has no socket" }))); + return; + } + resume(Effect.succeed({ status: response.statusCode ?? 0, socket, body })); + }); + }, + ); + request.once("error", (cause) => resume(Effect.fail(hostTestError(cause)))); + request.end(); + return Effect.sync(() => request.destroy()); + }); /** `true` once a fresh one-shot request to `port` is refused (reset) rather than answered. */ const isHttpRefused = (port: number) => @@ -1255,55 +1327,150 @@ it.live( { timeout: 60_000 }, ); -// Drives the deadline with a shortened `Network.ShutdownDrainDeadline` (the internal reference -// production always defaults to 10 s) rather than `TestClock`: this in-process owner's other -// timers (lease, idle, cooldown, readiness polling) share the same `Clock` through one layer -// composition, so virtualizing it for the deadline alone risks silently changing those unrelated -// subsystems too. The shortened real deadline keeps this test's own timing assertion-free in turn: -// ordering is observed (still open mid-drain, then cut once stop completes), never timed. +/** + * Drives `network.drain` directly, against a raw backend this test fully controls (not the real + * mail service), so a response can be gated open exactly across the drain boundary instead of + * merely completing fast. The deadline is an injected event (pass 3, F7): `ShutdownDrainDeadline` + * now holds an `Effect`, so the test supplies `Deferred.await(deadline)` and decides + * precisely when it fires, with no clock (real or virtual) involved at all. + */ it.live( - "cuts a hanging connection at the drain deadline, then stop completes", + "drains an in-flight response, refuses new connections, and cuts a hanging one only once the deadline completes (F7)", () => Effect.scoped( Effect.gen(function* () { - const { runtime, port } = yield* withRunningMail("stack-host-drain-deadline-"); - const agent = yield* Effect.acquireRelease( - Effect.sync(() => new Http.Agent({ keepAlive: true, maxSockets: 1 })), + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-network-drain-" }); + const state = yield* stateFor(`${root}/state`); + const stackId = "drain-network-test"; + yield* state.save({ + id: stackId, + runtime: "native" as const, + identity: { projectRoot: root, branchContext: "main", stackName: stackId }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + ports: [], + }); + const networkContext = yield* Layer.build( + Network.layer({ stackId, runtime: "native" }).pipe( + Layer.provide(Layer.succeed(StackNamespace.Service, state)), + ), + ); + const network = Context.get(networkContext, Network.Service); + + // A backend this test fully controls: each request is held open until its matching slot + // is released, so the first can be proven genuinely in flight across the drain boundary, + // and the second deliberately never completes on its own. + const slotA = { + received: yield* Deferred.make(), + release: yield* Deferred.make(), + }; + const slotB = { + received: yield* Deferred.make(), + release: yield* Deferred.make(), + }; + const slots = [slotA, slotB]; + const nextSlot = yield* Ref.make(0); + const services = yield* Effect.context(); + const server = Http.createServer((_request, response) => { + Effect.runForkWith(services)( + Effect.gen(function* () { + const index = yield* Ref.getAndUpdate(nextSlot, (value) => value + 1); + const slot = slots[index]; + if (slot === undefined) { + response.writeHead(503); + response.end(); + return; + } + yield* Deferred.succeed(slot.received, undefined); + yield* Deferred.await(slot.release); + response.writeHead(200, { "content-type": "text/plain" }); + response.end(`drained-ok-${index}`); + }), + ); + }); + const backendPort = yield* Effect.acquireRelease( + Effect.callback((resume) => { + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + resume( + Effect.succeed(address !== null && typeof address === "object" ? address.port : 0), + ); + }); + }), + () => + Effect.callback((resume) => { + server.close(() => resume(Effect.void)); + }), + ); + const namespace = yield* network.register({ + id: "backend", + endpoints: { + http: { + protocol: "http", + port: "auto", + backend: Effect.succeed({ host: "127.0.0.1", port: backendPort }), + enabled: Effect.succeed(true), + }, + }, + }); + const bound = yield* namespace.bind; + const port = bound.find((endpoint) => endpoint.name === "http")?.port; + if (port === undefined) return yield* Effect.die("Missing http endpoint"); + + const agentA = yield* Effect.acquireRelease( + Effect.sync(() => new Http.Agent({ keepAlive: false })), (agent) => Effect.sync(() => agent.destroy()), ); - // Establishes progress on the exact socket under test, not a different one: the listener - // has genuinely accepted and answered on it before drain, closing the race a bare - // client-side `connect` would otherwise leave against the server's own acceptance. - const first = yield* httpRoundTrip(port, agent); - expect(first.status, "the listener answers before the hang").toBe(200); - const shutdown = yield* Effect.forkScoped(shutdownOwner(runtime.access, false)); + const agentB = yield* Effect.acquireRelease( + Effect.sync(() => new Http.Agent({ keepAlive: false })), + (agent) => Effect.sync(() => agent.destroy()), + ); + // Both connections are established and in flight before drain starts: a connection made + // after accept has already closed would be refused outright, never "hanging". + const requestA = yield* Effect.forkScoped(httpRoundTrip(port, agentA)); + yield* Deferred.await(slotA.received); + const requestB = yield* Effect.forkScoped(httpRoundTrip(port, agentB)); + yield* Deferred.await(slotB.received); + + const deadline = yield* Deferred.make(); + const drain = yield* Effect.forkScoped( + network.drain.pipe( + Effect.provideService(Network.ShutdownDrainDeadline, Deferred.await(deadline)), + ), + ); yield* waitUntilRefused(port); - // A second, real in-flight request on that same connection, started only once drain is - // already refusing new ones, proves it keeps flowing mid-drain rather than merely sitting - // idle and unnoticed. - const second = yield* httpRoundTrip(port, agent); - expect(second.socket, "reuses the same established connection").toBe(first.socket); - expect(second.status, "a request on it still completes during drain").toBe(200); + + // The in-flight response completes with status 200 and its expected body during drain. + yield* Deferred.succeed(slotA.release, undefined); + const resultA = yield* Fiber.join(requestA); + expect(resultA.status, "the in-flight response completes during drain").toBe(200); + expect(resultA.body, "with its expected body").toBe("drained-ok-0"); + + // The second connection, deliberately never released, stays open: nothing but the test + // itself decides when the deadline elapses. expect( - second.socket.destroyed, - "the established connection survives into the drain window", - ).toBe(false); - // Deliberately left idle (no further request, the agent never closed) past this point, so - // the deadline cut — not a voluntary close — is what ends it. - yield* Fiber.join(shutdown).pipe( + requestB.pollUnsafe(), + "the hanging connection stays open before the deadline completes", + ).toBeUndefined(); + + yield* Deferred.succeed(deadline, undefined); + yield* Fiber.join(drain).pipe( Effect.timeoutOrElse({ - duration: "10 seconds", + duration: "5 seconds", orElse: () => Effect.fail( - new HostTestError({ message: "Stop did not complete after the drain deadline" }), + new HostTestError({ message: "Drain did not complete after the deadline" }), ), }), ); - expect(second.socket.destroyed, "the hanging connection is cut at the deadline").toBe(true); + const resultB = yield* Fiber.join(requestB).pipe(Effect.exit); + expect( + Exit.isFailure(resultB), + "the hanging connection is cut once the deadline completes", + ).toBe(true); }), - ).pipe( - Effect.provideService(Network.ShutdownDrainDeadline, "2 seconds"), - Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp)), - ), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 30_000 }, ); diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index e0365a2830..d97d4b5404 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -18,6 +18,7 @@ import { Fiber, Layer, Option, + PlatformError, Queue, Ref, Scope, @@ -44,7 +45,7 @@ import { projectSegmentFor } from "./identity/Identity.ts"; import * as Claims from "./namespace/Claims.ts"; import * as Owner from "./Owner.ts"; import { StackError, stackError, StackRpc, type RunCommandPayload } from "./Rpc.ts"; -import { engineUnreachable, makeHostGateway, resolveEngineTarget } from "./runtime/Container.ts"; +import { makeHostGateway, resolveEngineTarget } from "./runtime/Container.ts"; import * as StackNamespace from "./StackNamespace.ts"; import { sweepOrphans } from "./Sweep.ts"; import { makeCommandAttachments } from "./host/CommandAttachments.ts"; @@ -84,6 +85,28 @@ const hostError = (operation: string, cause: unknown, reason?: "runtime-unavaila ...(reason === undefined ? {} : { reason }), }); +/** + * Matches an engine CLI that is missing or reports a daemon that is not listening, never one + * that rejects the caller (for example on permissions): the one startup-diagnostic distinction + * `StackHostError`'s own `reason` has carried since before phase 2d, so `destroy`/`shutdown` + * (`effect.ts`) know to leave a stack registered for retry instead of surfacing a hard failure. + * Unrelated to abandonment's own cleanup-retry decision, which probes the engine directly instead + * of matching a message at all (pass 3, B). + */ +const engineUnreachableAtStartup = (cause: unknown): boolean => { + const message = cause instanceof Error ? cause.message : String(cause); + return ( + (cause instanceof Object && + "cause" in cause && + cause.cause instanceof PlatformError.PlatformError && + cause.cause.reason._tag === "NotFound" && + cause.cause.reason.method === "spawn") || + /cannot connect to the docker daemon|connection refused|connect: no such file or directory|error during connect:[^\n]*(?:docker daemon is not running|the system cannot find the file specified)/iu.test( + message, + ) + ); +}; + /** Binds the owner's loopback control listener on an OS-assigned port. */ export const bindControl = Effect.fn("StackHost.bindControl")(function* () { let rawServer: Http.Server | undefined; @@ -211,8 +234,9 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( }); const exit = yield* Deferred.make(); const gate = yield* Semaphore.make(1); + type ShutdownMode = "stop" | "destroy" | "abandon"; const current = yield* Ref.make< - { destroy: boolean; fiber: Fiber.Fiber } | undefined + { readonly mode: ShutdownMode; readonly fiber: Fiber.Fiber } | undefined >(undefined); const watchResponse = ( response: ReturnType, @@ -222,55 +246,78 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( responseClosed(response).pipe(Effect.andThen(Deferred.succeed(closed, undefined))), scope, ); + // Each stage settles and logs independently: one stage's failure (for example a retained + // native command's termination that cannot be confirmed) must never skip the others. + const settled = (label: string, stage: Effect.Effect) => + stage.pipe( + Effect.catchCause((cause) => Effect.logError(`Abandoned stack: ${label} failed`, cause)), + ); - const shutdown = Effect.fn("StackHost.shutdown")( - (destroy: boolean, response?: ReturnType) => - Effect.gen(function* () { - const fiber = yield* gate.withPermits(1)( - Effect.uninterruptibleMask(() => - Effect.gen(function* () { - const existing = yield* Ref.get(current); - if (existing !== undefined) { - if (existing.destroy && !destroy) return existing.fiber; - if (!existing.destroy && destroy) - return yield* new StackError({ - operation: "shutdown", - message: "Shutdown mode is already selected", - }); - return existing.fiber; - } - yield* owner.setDraining(true); - const responseClosedSignal = - response === undefined ? undefined : yield* Deferred.make(); - if (response !== undefined && responseClosedSignal !== undefined) - yield* watchResponse(response, responseClosedSignal); - let retiringAfterDestroyFailure = false; - const stopOwned = Effect.gen(function* () { - yield* attachments.stopAll; - yield* runner.cleanup; - yield* owner.namespace.stop; - }); - const finish = Effect.gen(function* () { - if (response !== undefined) { - if (responseClosedSignal === undefined) return; - yield* Effect.forkIn( - Deferred.await(responseClosedSignal).pipe( - Effect.andThen(closeConnections), - Effect.andThen(Deferred.succeed(exit, undefined)), - ), - scope, - ); - } else { - yield* closeConnections; - yield* Deferred.succeed(exit, undefined); - } + // Admits `mode`'s cleanup when nothing else is in flight: forks it, tracks it as `current` + // until it settles (success or failure alike), then clears `current` so a later claim — + // for example abandon after an in-flight stop or destroy — can be admitted again (pass 3, + // A). Only `stop` and `destroy` can actually fail; their own recovery (reopening admission) + // runs, and `current` is cleared, before `claim` below can ever admit anything else. + const begin = ( + mode: ShutdownMode, + response?: ReturnType, + ): Effect.Effect> => + Effect.gen(function* () { + yield* owner.setDraining(true); + const responseClosedSignal = + response === undefined ? undefined : yield* Deferred.make(); + if (response !== undefined && responseClosedSignal !== undefined) + yield* watchResponse(response, responseClosedSignal); + const finish = Effect.gen(function* () { + if (response !== undefined) { + if (responseClosedSignal === undefined) return; + yield* Effect.forkIn( + Deferred.await(responseClosedSignal).pipe( + Effect.andThen(closeConnections), + Effect.andThen(Deferred.succeed(exit, undefined)), + ), + scope, + ); + } else { + yield* closeConnections; + yield* Deferred.succeed(exit, undefined); + } + }); + const stopOwned = Effect.gen(function* () { + yield* attachments.stopAll; + yield* runner.cleanup; + yield* owner.namespace.stop; + }); + let retiringAfterDestroyFailure = false; + const body: Effect.Effect = + mode === "abandon" + ? Effect.gen(function* () { + // Attachment/command cleanup and namespace cleanup run concurrently; each + // retries until confirmed (or the engine is confirmed unavailable), and this + // waits for both before the owner is permitted to exit (pass 3, A). + const commandCleanup = settled( + "command cleanup", + Owner.retryUntilConfirmed( + attachments.stopAll.pipe(Effect.andThen(runner.cleanup)), + owner.engineAvailable, + ), + ); + const namespaceCleanup = settled( + "abandon namespace cleanup", + owner.namespace.abandon, + ); + yield* Effect.all([commandCleanup, namespaceCleanup], { + concurrency: "unbounded", + discard: true, }); - const cleanup = Effect.gen(function* () { - if (!destroy) { - yield* stopOwned; - yield* finish; - return; - } + yield* finish; + }) + : mode === "stop" + ? stopOwned.pipe( + Effect.andThen(finish), + Effect.mapError((cause) => stackError("shutdown", cause)), + ) + : Effect.gen(function* () { const destroyExit = yield* Effect.gen(function* () { yield* attachments.stopAll; yield* runner.cleanup; @@ -318,100 +365,94 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( retiringAfterDestroyFailure = true; yield* finish; return yield* Effect.failCause(destroyExit.cause); - }).pipe( - Effect.mapError((cause) => stackError("shutdown", cause)), - Effect.catchCause((cause) => - retiringAfterDestroyFailure - ? Effect.failCause(cause) - : owner - .setDraining(false) - // F6: drain may already have run (and so left the network's own - // admission closed) before this later stage failed; recovering it here - // keeps a start/restart that follows from refusing as still draining. - .pipe( - Effect.andThen(owner.namespace.recoverDraining), - Effect.andThen(gate.withPermits(1)(Ref.set(current, undefined))), - Effect.andThen(Effect.failCause(cause)), - ), - ), - ); - const fiber = yield* Effect.forkIn(cleanup, scope); - yield* Ref.set(current, { destroy, fiber }); - return fiber; + }).pipe(Effect.mapError((cause) => stackError("shutdown", cause))); + const cleanup = body.pipe( + Effect.catchCause((cause) => + mode !== "abandon" && !retiringAfterDestroyFailure + ? owner + .setDraining(false) + // F6: drain may already have run (and so left the network's own admission + // closed) before this later stage failed; recovering it here keeps a + // start/restart that follows from refusing as still draining. + .pipe( + Effect.andThen(owner.namespace.recoverDraining), + Effect.andThen(gate.withPermits(1)(Ref.set(current, undefined))), + Effect.andThen(Effect.failCause(cause)), + ) + : Effect.failCause(cause), + ), + ); + const fiber = yield* Effect.forkIn(cleanup, scope); + yield* Ref.set(current, { mode, fiber }); + return fiber; + }); + + // The one shutdown pipeline (pass 3, A): `stop`, `destroy` and `abandon` are claimed + // through the same gate and `current` record, never a separate join. If a shutdown is + // already in flight when abandonment is requested, this waits for it to settle (whatever + // its outcome) and then claims abandon fresh, unless that settlement was a confirmed + // successful destroy — ownership is already fully ended, so there is nothing left to + // abandon. Once abandon is claimed, nothing else can supersede it: it always joins. + const claim = ( + mode: ShutdownMode, + response?: ReturnType, + ): Effect.Effect => + Effect.gen(function* () { + while (true) { + const decision = yield* gate.withPermits(1)( + Effect.uninterruptibleMask(() => + Effect.gen(function* () { + const existing = yield* Ref.get(current); + if (existing === undefined) { + const fiber = yield* begin(mode, response); + return { _tag: "run", fiber } as const; + } + if (mode === "abandon") + return existing.mode === "abandon" + ? ({ _tag: "run", fiber: existing.fiber } as const) + : ({ + _tag: "await", + fiber: existing.fiber, + priorMode: existing.mode, + } as const); + if (existing.mode === "abandon") + return { _tag: "run", fiber: existing.fiber } as const; + if (existing.mode === "destroy" && mode === "stop") + return { _tag: "run", fiber: existing.fiber } as const; + if (existing.mode === "stop" && mode === "destroy") + return yield* new StackError({ + operation: "shutdown", + message: "Shutdown mode is already selected", + }); + return { _tag: "run", fiber: existing.fiber } as const; }), ), ); - yield* Fiber.join(fiber); - }).pipe(Effect.mapError((cause) => stackError("shutdown", cause))), - ); - // F6: a registration-loss poll drives this, never an RPC caller, so there is no response to - // watch and no registration left to touch. An in-flight shutdown already owns `current`; - // abandonment only joins it instead of racing its own cleanup against it. - // Each stage settles and logs independently: one stage's failure (for example a retained - // native command's termination that cannot be confirmed) must never skip the others (F4). - const settled = (label: string, stage: Effect.Effect) => - stage.pipe( - Effect.catchCause((cause) => Effect.logError(`Abandoned stack: ${label} failed`, cause)), + if (decision._tag === "run") { + yield* Fiber.join(decision.fiber); + return; + } + // "await": a prior stop or destroy, whether still running or already settled, is + // never cleared for its own sake (a repeated `shutdown` call must keep rejoining it + // for free); only abandon taking over after it clears the slot, so this loop's next + // pass can claim fresh. + const outcome = yield* Fiber.join(decision.fiber).pipe(Effect.exit); + if (decision.priorMode === "destroy" && Exit.isSuccess(outcome)) return; + yield* gate.withPermits(1)( + Ref.update(current, (value) => (value?.fiber === decision.fiber ? undefined : value)), + ); + } + }); + const shutdown = ( + destroy: boolean, + response?: ReturnType, + ) => + claim(destroy ? "destroy" : "stop", response).pipe( + Effect.mapError((cause) => stackError("shutdown", cause)), ); - const abandonResources = settled("abandon namespace cleanup", owner.namespace.abandon); - // F5: a retained native command's termination can still be failing when this runs; retried - // under the same policy as every other abandonment cleanup stage (confirmed, or a confirmed - // permanently unreachable engine) before the owner is permitted to exit. Namespace cleanup - // below still runs on its own regardless of how long this takes. - const retainedCommandCleanup = settled( - "command runner cleanup", - runner.cleanup.pipe(Effect.retry(Owner.abandonCleanupSchedule)), - ); - const abandon = Effect.fn("StackHost.abandon")(() => - gate - .withPermits(1)( - Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - const existing = yield* Ref.get(current); - if (existing !== undefined) return existing; - yield* owner.setDraining(true); - const cleanup = restore( - Effect.gen(function* () { - yield* settled("stopping attachments", attachments.stopAll); - yield* retainedCommandCleanup; - yield* abandonResources; - }), - ).pipe( - Effect.andThen(closeConnections), - Effect.andThen(Deferred.succeed(exit, undefined)), - Effect.asVoid, - ); - const fiber = yield* Effect.forkIn(cleanup, scope); - const record = { destroy: false, fiber }; - yield* Ref.set(current, record); - return record; - }), - ), - ) - .pipe( - Effect.flatMap((record) => - Fiber.join(record.fiber).pipe( - Effect.exit, - // `record.fiber` may be an in-flight normal shutdown, typed to report `StackError`; - // abandonment never fails the caller (the registration poll), so it is logged. - Effect.tap((joined) => - Exit.isFailure(joined) - ? Effect.logError("Stack shutdown failed", joined.cause) - : Effect.void, - ), - Effect.flatMap((joined) => - // F4: a joined destroy only confirmed both data removal and reservation release - // when it actually succeeded; a failed destroy can have fallen back to a plain - // stop (which preserves both) or failed before removing anything, so either way - // F6 cleanup still has to run. Inspecting the outcome, not the requested mode, - // is what tells the two apart; running it again on an already-confirmed destroy - // is redundant but safe, since the shared cleanup path is idempotent. - record.destroy && Exit.isSuccess(joined) ? Effect.void : abandonResources, - ), - ), - ), - ), - ); + // F6: a registration-loss poll drives this, never an RPC caller, so there is no response to + // watch and no registration left to touch; `claim` never actually fails for this mode. + const abandon = claim("abandon").pipe(Effect.ignore); const handlers = StackRpc.of({ ...owner.handlers, runCommand: (input: RunCommandPayload) => attachments.run(input), @@ -466,7 +507,7 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( access, serve, shutdown, - abandon: abandon(), + abandon, closeConnections, exit, }; @@ -556,14 +597,6 @@ export const runStackHost = Effect.fn("StackHost.run")( yield* fs.makeDirectory(dataRootPath, { recursive: true }); const dataRoot = yield* fs.realPath(dataRootPath); const project = projectSegmentFor(saved.identity, path); - const asHostError = (cause: unknown) => - hostError( - "startup-cleanup", - cause, - cause instanceof Object && "reason" in cause && cause.reason === "engine-unavailable" - ? "runtime-unavailable" - : undefined, - ); // Resolved once, here, for this owner's whole lifetime: the container runtime, the // storage helpers, the host-gateway probes and reconcile below all share this one // target instead of each resolving (and so potentially disagreeing on) their own. @@ -576,12 +609,18 @@ export const runStackHost = Effect.fn("StackHost.run")( hostError( "startup-cleanup", cause, - engineUnreachable(cause) ? "runtime-unavailable" : undefined, + engineUnreachableAtStartup(cause) ? "runtime-unavailable" : undefined, ), ), ); yield* Owner.sweepContainers(state, saved, dataRoot, engineTarget).pipe( - Effect.mapError(asHostError), + Effect.mapError((cause) => + hostError( + "startup-cleanup", + cause, + engineUnreachableAtStartup(cause) ? "runtime-unavailable" : undefined, + ), + ), ); const hostGateway = yield* makeHostGateway; const services = yield* Layer.build( diff --git a/packages/stack/src/runtime/Container.integration.test.ts b/packages/stack/src/runtime/Container.integration.test.ts index 9d3d0bc3ca..11702702ff 100644 --- a/packages/stack/src/runtime/Container.integration.test.ts +++ b/packages/stack/src/runtime/Container.integration.test.ts @@ -25,11 +25,12 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; import { HttpClient } from "effect/unstable/http"; import { + ContainerError, ContainerLaunchError, DOCKER_HOST_ALIAS, - engineUnreachable, makeContainerRuntime, makeHostGateway, + probeEngineReachable, resolveEngineTarget, type ContainerProcess, type EngineTarget, @@ -1494,37 +1495,29 @@ describe("container process adapter", () => { ), ); - it.live( - "classifies a real connection refusal to the engine as permanently unavailable (F3)", - () => - Effect.gen(function* () { - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - // A real connection attempt, not a mocked one: nothing listens on this loopback port, so - // the docker CLI's own refusal is what `errorFor` classifies, at its one true source. - const failure = yield* resolveEngineTarget(spawner).pipe(Effect.flip); - expect( - engineUnreachable(failure), - "a real daemon connection refusal classifies as engine-unavailable", - ).toBe(true); - }).pipe( - Effect.provide( - Layer.merge( - NodeServices.layer, - ConfigProvider.layer( - ConfigProvider.fromEnvRecord({ DOCKER_HOST: "tcp://127.0.0.1:1" }), - ), - ), - ), - ), + it.live("probes a real connection refusal to the engine as unreachable (pass 3, B)", () => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + // A real connection attempt, not a mocked one: nothing listens on this loopback port, so + // the probe observes the docker CLI's own refusal directly, never a classified message. + const target: EngineTarget = { + engine: "docker", + argv: ["--host", "tcp://127.0.0.1:1"], + daemonId: "probe-test", + }; + expect( + yield* probeEngineReachable(spawner, target), + "a real daemon connection refusal resolves unreachable", + ).toBe(false); + }).pipe(Effect.provide(NodeServices.layer)), ); - it.live("classifies a real missing engine binary as permanently unavailable (F3)", () => + it.live("probes a real missing engine binary as unreachable (pass 3, B)", () => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const fs = yield* FileSystem.FileSystem; // A directory deliberately containing no `docker` binary, real `PATH` lookup included: the - // engine's own spawn failure, not a stubbed `ChildProcessSpawner`, is what `errorFor` - // classifies. + // probe observes the engine's own spawn failure directly, never a stubbed spawner. const emptyBinDir = yield* fs.makeTempDirectoryScoped({ prefix: "engine-missing-bin-" }); // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the spawned child inherits PATH; this is not application config. const originalPath = process.env.PATH; @@ -1539,16 +1532,44 @@ describe("container process adapter", () => { process.env.PATH = originalPath; }), ); - const failure = yield* resolveEngineTarget(spawner).pipe(Effect.flip); + const target: EngineTarget = { engine: "docker", argv: [], daemonId: "probe-test" }; expect( - engineUnreachable(failure), - "a real spawn ENOENT for the missing docker binary classifies as engine-unavailable", - ).toBe(true); - }).pipe( - Effect.provide( - Layer.merge(NodeServices.layer, ConfigProvider.layer(ConfigProvider.fromEnvRecord({}))), - ), - ), + yield* probeEngineReachable(spawner, target), + "a real spawn ENOENT for the missing docker binary resolves unreachable", + ).toBe(false); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live( + "keeps retrying a daemon-relayed failure, such as a registry pull refusal, while the real engine stays reachable (pass 3, B)", + () => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const target = yield* resolveEngineTarget(spawner); + const attempts = yield* Ref.make(0); + // Simulates a registry rejecting a pull while the daemon itself answers normally: this + // must never be mistaken for an unavailable engine (F4), so `retryUntilConfirmed` keeps + // retrying for as long as the probe keeps confirming the daemon is reachable. + const daemonRelayedFailure = Ref.updateAndGet(attempts, (count) => count + 1).pipe( + Effect.andThen( + Effect.fail( + new ContainerError({ + operation: "pull", + message: + "Error response from daemon: pull access denied for supabase/does-not-exist-image, repository does not exist or may require 'docker login'", + }), + ), + ), + ); + const outcome = yield* Owner.retryUntilConfirmed( + daemonRelayedFailure, + probeEngineReachable(spawner, target), + ).pipe(Effect.timeoutOption("1.5 seconds")); + expect(Option.isNone(outcome), "never confirmed, and the engine probe never gave up").toBe( + true, + ); + expect(yield* Ref.get(attempts)).toBeGreaterThan(1); + }).pipe(Effect.provide(NodeServices.layer)), ); it.live( diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index 9d28c360f4..dfb51e56a6 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -12,7 +12,6 @@ import { Fiber, Option, Path, - PlatformError, Ref, Schedule, Schema, @@ -30,7 +29,6 @@ export class ContainerError extends Data.TaggedError("ContainerError")<{ readonly operation: string; readonly message: string; readonly cause?: unknown; - readonly reason?: "engine-unavailable"; }> {} interface ContainerSpec { @@ -93,44 +91,12 @@ export interface ContainerRuntime { ) => Effect.Effect; } -/** A spawn failure for a missing engine CLI binary; it carries no message pattern to test. */ -const isMissingEngineBinary = (cause: unknown): boolean => - cause instanceof PlatformError.PlatformError && - cause.reason._tag === "NotFound" && - cause.reason.method === "spawn"; - -/** - * Matches an engine CLI that is missing or reports a daemon that is not listening, not one that - * rejects the caller. Windows' `error during connect` also wraps authentication and TLS failures, - * so only its refused or missing-endpoint causes match. Matches regardless of which wrapping error - * type carried the message (a bare `ContainerError`, or another operation's error whose message - * preserves the underlying CLI failure text, such as a database storage cleanup failure). - */ -const engineUnreachableMessage = (message: string): boolean => - /cannot connect to the docker daemon|connection refused|connect: no such file or directory|error during connect:[^\n]*(?:docker daemon is not running|the system cannot find the file specified)/iu.test( - message, - ); - -/** - * Whether `cause` (a spawn failure for a missing engine binary) or `message` (the engine CLI's - * own unreachable-daemon phrasing) signals a permanently unavailable engine (F3). Classified once, - * here, at every engine invocation boundary (`errorFor`, below), and carried onward through - * wrapping as the error's own `reason` rather than re-matched; a caller that only ever sees a - * message (database storage, which invokes the engine CLI directly rather than through this - * module's `ContainerError`) calls this too, at its own equivalent source. - */ -export const isEngineUnavailable = (cause: unknown, message: string): boolean => - isMissingEngineBinary(cause) || engineUnreachableMessage(message); - -const errorFor = (operation: string, cause: unknown): ContainerError => { - const message = cause instanceof Error ? cause.message : String(cause); - return new ContainerError({ +const errorFor = (operation: string, cause: unknown): ContainerError => + new ContainerError({ operation, - message, + message: cause instanceof Error ? cause.message : String(cause), cause, - ...(isEngineUnavailable(cause, message) ? { reason: "engine-unavailable" as const } : {}), }); -}; /** Labels containers this run creates, when `SUPABASE_STACK_TEST_RUN` is set. */ const testRunLabelArgs = readTestRunLabelArgs.pipe( @@ -150,9 +116,6 @@ const transientPullFailure = (error: ContainerError) => error.message, ); -/** Classified once, at construction (`errorFor`), and simply read back here. */ -export const engineUnreachable = (error: ContainerError) => error.reason === "engine-unavailable"; - /** Runs one engine CLI invocation outside any pinned target, for resolving that target itself. */ const runRaw = ( spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], @@ -232,9 +195,24 @@ export const resolveEngineTarget = Effect.fn("Container.resolveEngineTarget")(fu return { engine: "docker", argv, daemonId }; }); -/** A pull worth retrying: rate-limited or a dropped connection, never an unreachable engine. */ -const retryablePull = (error: ContainerError) => - (rateLimited(error) || transientPullFailure(error)) && !engineUnreachable(error); +/** A pull worth retrying: rate-limited or a dropped connection. */ +const retryablePull = (error: ContainerError) => rateLimited(error) || transientPullFailure(error); + +/** + * Probes the pinned engine target directly, with the cheapest real call (`info`), rather than + * classifying a failure's message (pass 3, B): a spawn failure (missing binary) or the daemon + * endpoint refusing the connection both resolve `false`; any other failure (a daemon-relayed + * error, for example a registry rejecting a pull while the daemon itself is up) never reaches + * this probe and so never resolves `false` through it. + */ +export const probeEngineReachable = ( + spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], + target: EngineTarget, +): Effect.Effect => + runRaw(spawner, [...target.argv, "info", "--format", "{{.ID}}"]).pipe( + Effect.map((id) => id.trim().length > 0), + Effect.orElseSucceed(() => false), + ); const PULL_MAX_RETRIES = 4; @@ -771,7 +749,6 @@ export const makeContainerRuntime = (options: { operation: error.operation, message: `${error.message} (container name ${name})`, cause: error, - ...(error.reason === undefined ? {} : { reason: error.reason }), }), ), Effect.catchTag("TimeoutError", () => @@ -1074,10 +1051,52 @@ export const removeContainerById = Effect.fn("Container.removeContainerById")( }), ).pipe( Effect.timeout("30 seconds"), - // `errorFor` already classifies at construction; nothing further to reclassify here. Effect.mapError((cause) => cause instanceof ContainerError ? cause : errorFor("cleanup", cause), ), ); }), ); + +/** + * Lists every container carrying this stack's identity label, through the pinned engine: + * service containers and every storage helper alike (per-instance and shared), independent of + * any in-memory registry's own bookkeeping (pass 3, C). + */ +const listStackContainers = Effect.fn("Container.listStackContainers")(function* (options: { + readonly target: EngineTarget; + readonly stackId: string; +}) { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const output = yield* runRaw(spawner, [ + ...options.target.argv, + "ps", + "--all", + "--quiet", + "--no-trunc", + "--filter", + `label=com.supabase.stack=${options.stackId}`, + ]); + return output + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); +}); + +/** + * Removes every container carrying this stack's identity label and reports how many remain + * (normally 0): the registration-independent confirming sweep abandonment uses instead of + * depending on any helper registry's own bookkeeping (pass 3, C). A container that disappears + * between listing and removal is not an error, matching {@link removeContainerById}. + */ +export const removeStackContainers = Effect.fn("Container.removeStackContainers")( + function* (options: { readonly target: EngineTarget; readonly stackId: string }) { + const ids = yield* listStackContainers(options); + yield* Effect.forEach(ids, (id) => removeContainerById({ target: options.target, id }), { + concurrency: "unbounded", + discard: true, + }); + const remaining = yield* listStackContainers(options); + return remaining; + }, +); diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index e05c576e0f..8b7b58c295 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -18,11 +18,7 @@ import { failureMessage } from "../internal/failure-message.ts"; import { testRunLabelArgs as readTestRunLabelArgs } from "../internal/test-run-label.ts"; import type * as Claims from "../namespace/Claims.ts"; import * as Publication from "../namespace/Publication.ts"; -import { - isEngineUnavailable, - type ContainerRuntime, - type EngineTarget, -} from "../runtime/Container.ts"; +import type { ContainerRuntime, EngineTarget } from "../runtime/Container.ts"; import { composeProjectFor } from "../runtime/ContainerName.ts"; import type { DatabaseRuntime } from "../services/Database.ts"; import { @@ -65,8 +61,6 @@ export class DockerDatabaseStorageError extends Schema.TaggedError Schema.is(DockerDatabaseStorageError)(cause) ? cause @@ -112,16 +102,8 @@ const errorFor = (operation: string, cause: unknown) => operation: cause.operation, message: cause.message, cause, - ...(isEngineUnavailable(cause, cause.message) ? { reason: "engine-unavailable" } : {}), }) - : new DockerDatabaseStorageError({ - operation, - message: failureMessage(cause), - cause, - ...(isEngineUnavailable(cause, failureMessage(cause)) - ? { reason: "engine-unavailable" } - : {}), - }); + : new DockerDatabaseStorageError({ operation, message: failureMessage(cause), cause }); const parseMajor = (version: string): number | undefined => { const major = Number(version.trim().split(".")[0]); From 5baf334dcfde18a3209e70790664882b15a9d26d Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 11:32:33 +0200 Subject: [PATCH 038/142] test(stack): cover registration loss mid-drain and retry-while-reachable abandonment Adds the missing regression coverage for abandonment re-checking registration after a signal-driven stop's drain completes, and for retrying a daemon-relayed failure while the engine stays reachable, closing the gaps a review surfaced in the shutdown-drain and registration-loss abandonment phase. Includes the effect v4 API fixes the phase's own retry code needed to compile, a subscribe-before-signal fix in the new drain-gate test fixture to remove a TOCTOU race under load, and the lint allowlist entry the fixture needs. --- .oxlintrc.json | 3 +- packages/stack/src/HostProcess.ts | 11 +- packages/stack/src/HttpProxy.ts | 2 +- packages/stack/src/Network.ts | 16 +- packages/stack/src/Orchestrator.ts | 2 +- packages/stack/src/Owner.ts | 64 ++--- packages/stack/src/Proxy.ts | 2 +- .../stack/src/Service.integration.test.ts | 2 +- packages/stack/src/Service.ts | 2 +- ...ost.container-shutdown.integration.test.ts | 226 +++++++++++++++++- packages/stack/src/StackHost.ts | 63 +++-- .../stack/src/lifetime.integration.test.ts | 4 +- .../src/runtime/Container.integration.test.ts | 35 ++- packages/stack/src/runtime/Container.ts | 33 ++- .../services/Functions.integration.test.ts | 6 +- packages/stack/tests/gated-drain-fixture.ts | 114 +++++++++ .../tests/short-registration-poll-fixture.ts | 2 +- 17 files changed, 481 insertions(+), 106 deletions(-) create mode 100644 packages/stack/tests/gated-drain-fixture.ts diff --git a/.oxlintrc.json b/.oxlintrc.json index f8e74bf71b..f026649b47 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -383,7 +383,8 @@ "packages/stack/tests/effect-timeout.fixture.ts", "packages/stack/tests/host-process-fixture.ts", "packages/stack/tests/failing-owner-fixture.ts", - "packages/stack/tests/short-registration-poll-fixture.ts" + "packages/stack/tests/short-registration-poll-fixture.ts", + "packages/stack/tests/gated-drain-fixture.ts" ], "rules": { "no-restricted-imports": [ diff --git a/packages/stack/src/HostProcess.ts b/packages/stack/src/HostProcess.ts index c3754bedbb..dbda07c3db 100644 --- a/packages/stack/src/HostProcess.ts +++ b/packages/stack/src/HostProcess.ts @@ -346,6 +346,8 @@ export interface LaunchOptions { readonly cacheRoot: string; readonly stackId: string; readonly entrypoint?: string; + /** Extra positional arguments a non-default `entrypoint` reads, ahead of `register`. */ + readonly entrypointArgs?: ReadonlyArray; /** Registers this definition once the spawned owner holds the lease. */ readonly register?: SavedStack; /** Ties a spawned owner to the enclosing scope, whose closure destroys the stack. */ @@ -463,7 +465,14 @@ const spawnOwner = Effect.fn("HostProcess.spawnOwner")(function* ( try: () => { const started = spawn( process.execPath, - [entrypoint, options.stateRoot, options.cacheRoot, options.stackId, ...register], + [ + entrypoint, + options.stateRoot, + options.cacheRoot, + options.stackId, + ...(options.entrypointArgs ?? []), + ...register, + ], { cwd: process.cwd(), detached: true, diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index 1348849514..0fae17ab2c 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -47,7 +47,7 @@ export interface HttpProxy { readonly setRoutes: (routes: ReadonlyArray) => Effect.Effect; /** Stops accepting new connections on the listener; established connections keep flowing. */ readonly stopAccepting: Effect.Effect; - /** Reverses `stopAccepting` (F6): recovers a listener left refusing after a failed drain. */ + /** Reverses `stopAccepting`: recovers a listener left refusing after a failed drain. */ readonly resumeAccepting: Effect.Effect; /** The count of established client connections, observable until it reaches 0. */ readonly outstandingConnections: SubscriptionRef.SubscriptionRef; diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index 21e3712e02..8cd745f1fe 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -20,9 +20,9 @@ import { makeHttpProxy, type HttpProxy, type HttpRoute } from "./HttpProxy.ts"; export type NetworkRuntime = "native" | "docker"; /** - * The shutdown-drain deadline (F5): established connections keep flowing until this event - * completes. An injectable event rather than a duration (pass 3, F7), so a test can gate exactly - * when it fires through a `Deferred` instead of racing a shortened real timer; production always + * The shutdown-drain deadline: established connections keep flowing until this event + * completes. An injectable event rather than a duration, so a test can gate when it fires + * through a `Deferred` instead of racing a shortened real timer; production always * defaults to a plain 10-second sleep. Internal only, following the `RegistrationCheckInterval` * pattern: production never reads it from an environment variable or `Config`. */ @@ -99,14 +99,14 @@ export interface Interface { readonly endpoints: Readonly>; }) => Effect.Effect; /** - * Shutdown drain (F5): closes accept on every stack listener, public and dependency alike, then + * Shutdown drain: closes accept on every stack listener, public and dependency alike, then * lets established connections keep flowing until each reaches 0 or `ShutdownDrainDeadline` * elapses, whichever comes first. At the deadline every remaining connection is cut. Listener * scopes are untouched; the caller closes them afterward through the ordinary stop/destroy path. */ readonly drain: Effect.Effect; /** - * Recovers from a shutdown that failed after {@link drain} began (F6): resets the drain flag + * Recovers from a shutdown that failed after {@link drain} began: resets the drain flag * `bind` checks and resumes accepting on every currently tracked listener, so a start/restart * that follows a failed shutdown binds normally instead of refusing as still draining forever. * Paired with the owner's own admission recovery; never called while a shutdown might still @@ -141,7 +141,7 @@ const makeNetwork = (options: { // Set under `gate` by `drain`, before it snapshots `listeners`; checked under the same gate by // `bind`, so a bind that hasn't yet acquired the gate when drain starts either completes // before drain's snapshot (and so is captured by it) or observes this and refuses outright - // (F2) — never slips a new, untracked listener past both the snapshot and the accept cut. + // so neither slips a new, untracked listener past both the snapshot and the accept cut. const draining = yield* Ref.make(false); // Every bound listener (the shared API proxy under "api", every dedicated endpoint under // "id:name"), tracked only for shutdown drain; registration and deregistration are tied to the @@ -440,8 +440,8 @@ const makeNetwork = (options: { const drain = Effect.fn("Network.drain")(function* () { const deadline = yield* ShutdownDrainDeadline; // Setting `draining` and snapshotting `listeners` atomically under the same gate `bind` - // holds for its whole call (F2) means no bind can complete after this snapshot without - // either being captured by it or observing `draining` and refusing before creating anything. + // holds for its whole call means no bind can complete after this snapshot without either + // being captured by it or observing `draining` and refusing before creating anything. const handles = yield* gate.withPermits(1)( Ref.set(draining, true).pipe( Effect.andThen(Ref.get(listeners)), diff --git a/packages/stack/src/Orchestrator.ts b/packages/stack/src/Orchestrator.ts index bec2cf36c0..544d4296ba 100644 --- a/packages/stack/src/Orchestrator.ts +++ b/packages/stack/src/Orchestrator.ts @@ -84,7 +84,7 @@ export interface RegisteredInstance { * Releases this instance's network reservations once its data is confirmed removed: destroy's * own step, run right after `core.removeData` succeeds. Abandonment never calls it — a stale * reservation stays for `Ports.ts`'s lazy reclamation once the holder's registration is - * confirmed gone, exactly the abandonment trigger, so abandonment has nothing to release itself. + * confirmed gone, the abandonment trigger, so abandonment has nothing to release itself. */ readonly release: Effect.Effect; readonly hasEndpoint: boolean; diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index deabcefcec..eaeccdcc1c 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -182,21 +182,19 @@ export interface Interface { */ readonly destroy: Effect.Effect; /** - * Ends ownership after a confirmed-gone registration (F6): drains, stops every workload and - * removes what it created through the same registration-independent `cleanupResources` path as - * destroy. Never reads or writes the registration, which is already gone, and never releases - * port reservations either: a stale one stays for `Ports.ts`'s own lazy reclamation once the - * holder's registration is confirmed gone — exactly this trigger. A leftover resource past the - * engine-unreachable backstop is logged, not thrown. + * Ends ownership after a confirmed-gone registration: drains, stops every workload, and + * removes what it created through the same registration-independent cleanup path as destroy, + * without touching the registration or releasing port reservations. A leftover resource past + * the engine-unreachable backstop is logged, not thrown. */ readonly abandon: Effect.Effect; /** - * Recovers network admission after a shutdown that failed once drain had already begun (F6): - * see {@link Network.Interface.recoverDrain}. + * Recovers network admission after a shutdown that failed once drain had already begun; see + * {@link Network.Interface.recoverDrain}. */ readonly recoverDraining: Effect.Effect; }; - /** Direct probe of the pinned engine target; always `true` for a native stack (pass 3, B). */ + /** Direct probe of the pinned engine target; always `true` for a native stack. */ readonly engineAvailable: Effect.Effect; readonly setDraining: (draining: boolean) => Effect.Effect; readonly getServing: Effect.Effect; @@ -231,9 +229,19 @@ const serviceError = const rpcError = (operation: string) => Effect.mapError((cause: unknown) => stackError(operation, cause)); +/** + * The first backoff delay {@link retryUntilConfirmed} sleeps between attempts, doubling up to 5 + * seconds. Internal only: a test overrides this reference to observe several attempts without + * waiting through a production-length backoff, never through an env var or `Config`. + */ +export const RetryUntilConfirmedInitialDelay = Context.Reference( + "@supabase/stack/RetryUntilConfirmedInitialDelay", + { defaultValue: () => Duration.millis(200) }, +); + /** * Retries `operation` with a bounded, doubling backoff until it succeeds or `engineAvailable` - * resolves `false` (pass 3, B): engine availability is a direct probe, never a message or cause + * resolves `false`: engine availability is a direct probe, never a message or cause * classification, so this is the one place every abandonment cleanup stage decides whether a * failure is worth retrying. A `false` probe fails with the operation's own last error; an * interruption or defect is never retried. @@ -243,7 +251,7 @@ export const retryUntilConfirmed = ( engineAvailable: Effect.Effect, ): Effect.Effect => Effect.gen(function* () { - let delay = Duration.millis(200); + let delay = Duration.fromInputUnsafe(yield* RetryUntilConfirmedInitialDelay); while (true) { const result = yield* Effect.exit(operation); if (Exit.isSuccess(result)) return result.value; @@ -317,14 +325,14 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { const network = yield* Network.Service; const portReservations = yield* PortReservations.Service; // A native stack has no engine to lose, so every cleanup failure is retried forever; a docker - // stack probes the pinned target directly (pass 3, B), never by classifying a failure's cause. + // stack probes the pinned target directly, never by classifying a failure's cause. const engineAvailable: Effect.Effect = options.engineTarget === undefined ? Effect.succeed(true) : Container.probeEngineReachable(spawner, options.engineTarget); // Shared per-launch environment-file scratch directory (Container.ts), owned by this stack as a - // whole rather than any one instance; registration-independent so destroy and abandonment (F7) - // both reach it without depending on `Registry.remove`, which never runs during abandonment. + // whole rather than any one instance; registration-independent so destroy and abandonment both + // reach it without depending on `Registry.remove`, which never runs during abandonment. const removeContainerEnvRoot = fs .remove(path.join(options.root, Paths.CONTAINER_ENV_DIRNAME), { recursive: true, force: true }) .pipe(Effect.mapError(serviceError("cleanup"))); @@ -511,7 +519,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { // neither reads or writes the registration. Network reservations are never part of this: they // are destroy's own separate `release` step (`Orchestrator.destroy`), never abandonment's, // since a stale reservation stays for `Ports.ts`'s lazy reclamation once the holder's - // registration is confirmed gone — exactly the abandonment trigger. + // registration is confirmed gone, the abandonment trigger. const cleanupResources = (context: ServiceInstanceContext) => recipe.definition.removeData(context); const core = yield* makeService( @@ -866,7 +874,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { handlers, getStackCredentials, namespace: { - // Shutdown drain (F5): accept closes on every listener before any service stops, and the + // Shutdown drain: accept closes on every listener before any service stops, and the // listener scopes only close afterward, through stop/destroy's ordinary teardown below. stop: network.drain.pipe( Effect.andThen(orchestrator.stopNamespace), @@ -919,7 +927,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { definitionGate.withPermits(1), Effect.withSpan("Owner.destroyNamespace"), ), - // F6: settles admitted definition work under the same gate as stop/destroy first. Never + // Settles admitted definition work under the same gate as stop/destroy first. Never // reads or writes the registration (already confirmed gone), so every step below is // best-effort and logs rather than fails; the caller must still be able to exit. abandon: network.drain.pipe( @@ -940,10 +948,12 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ), ), ), - // Registration-independent confirming sweep (pass 3, C): every container carrying this - // stack's identity label, found through the pinned engine directly rather than through - // any in-memory helper registry's own bookkeeping — covers a shared storage helper the - // per-instance cleanup above never removes on its own. + // Registration-independent confirming sweep: every container carrying this stack's + // identity and data-root labels, found through the pinned engine directly rather than + // through any in-memory helper registry's own bookkeeping — covers a shared storage + // helper the per-instance cleanup above never removes on its own. The data-root label + // keeps this scoped to this owner's own containers even when another owner shares the + // same stack id under a different root. Effect.andThen( options.engineTarget === undefined ? Effect.void @@ -951,6 +961,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { Container.removeStackContainers({ target: options.engineTarget, stackId, + stackRoot: path.resolve(options.root), }).pipe( Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), Effect.mapError(serviceError("cleanup")), @@ -973,12 +984,11 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ), ), ), - // Filesystem-only, so always retried to confirmation regardless of engine availability - // (F7): its own `force: true` tolerates an already-gone directory, never a transient - // failure. Port reservations are never released here (simplification over F2): a stale - // reservation stays for `Ports.ts`'s lazy reclamation once the holder's registration is - // confirmed gone — exactly the abandonment trigger — so abandonment never touches - // `Ports.ts`'s registry at all, even once the state root itself no longer exists. + // Filesystem-only, so always retried to confirmation regardless of engine availability: + // its own `force: true` tolerates an already-gone directory, never a transient failure. + // Port reservations are never released here: a stale reservation stays for `Ports.ts`'s + // own lazy reclamation once the holder's registration is confirmed gone, the abandonment + // trigger, so abandonment never touches `Ports.ts`'s registry at all. Effect.andThen( retryUntilConfirmed(removeContainerEnvRoot, Effect.succeed(true)).pipe( Effect.catch((cause) => diff --git a/packages/stack/src/Proxy.ts b/packages/stack/src/Proxy.ts index 02b8cb8241..15c914e78d 100644 --- a/packages/stack/src/Proxy.ts +++ b/packages/stack/src/Proxy.ts @@ -24,7 +24,7 @@ export interface TcpListener { ) => Effect.Effect; /** Stops accepting new connections on the listener; established connections keep flowing. */ readonly stopAccepting: Effect.Effect; - /** Reverses `stopAccepting` (F6): recovers a listener left refusing after a failed drain. */ + /** Reverses `stopAccepting`: recovers a listener left refusing after a failed drain. */ readonly resumeAccepting: Effect.Effect; /** The count of established client connections, observable until it reaches 0. */ readonly outstandingConnections: SubscriptionRef.SubscriptionRef; diff --git a/packages/stack/src/Service.integration.test.ts b/packages/stack/src/Service.integration.test.ts index 50d9082718..9a492d9b42 100644 --- a/packages/stack/src/Service.integration.test.ts +++ b/packages/stack/src/Service.integration.test.ts @@ -635,7 +635,7 @@ describe("service execution", () => { ); it.live( - "reports confirmed termination when removeData cleans up a session a failed stop retained (F1)", + "reports confirmed termination when removeData cleans up a session a failed stop retained", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index ec4d6a567b..0f37ee868d 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -492,7 +492,7 @@ export const makeService = ( const leftover = yield* Ref.get(current); if (leftover !== undefined) { yield* stopRecord(leftover, true); - // F1: a previous stop attempt may have failed and reported `StopFailed`, leaving the + // A previous stop attempt may have failed and reported `StopFailed`, leaving the // reducer in `Stopping` forever; this retry's own confirmed termination must reach it // too, the same way `stop`'s own successful path does, or the reducer (and anything // gated on it, such as `NetworkNamespace.release`'s endpoint check) never learns. diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index 00509807b4..a91bc68261 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -5,6 +5,7 @@ import { Crypto, Effect, FileSystem, + Fiber, Layer, Path, Redacted, @@ -12,6 +13,7 @@ import { Stream, } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import * as Net from "node:net"; // oxlint-disable-line effecttsgo/node-builtin-import -- forces an outstanding connection during a gated drain. import { fileURLToPath } from "node:url"; import * as StackNamespace from "./StackNamespace.ts"; import { @@ -29,6 +31,9 @@ import { noContainerClaims } from "../tests/claims.ts"; const shortRegistrationPollFixture = fileURLToPath( new URL("../tests/short-registration-poll-fixture.ts", import.meta.url), ); +const gatedDrainFixture = fileURLToPath( + new URL("../tests/gated-drain-fixture.ts", import.meta.url), +); const helperImage = "public.ecr.aws/docker/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; @@ -100,7 +105,7 @@ const startHost = (stateRoot: string, cacheRoot: string, stackId: string, projec }); it.live.skipIf(process.platform === "win32")( - "stops only containers owned by the same stack id and data root", + "stops, destroys, and abandons only containers owned by the same stack id and data root", () => Effect.scoped( Effect.gen(function* () { @@ -237,13 +242,108 @@ it.live.skipIf(process.platform === "win32")( activeB = undefined; expect(yield* containers(stackId, dataB), "destroy removes B containers").toEqual([]); expect(yield* (yield* stateFor(rootB)).read(stackId)).toBeUndefined(); + + // Abandonment phase: two fresh owners share this id again, each under its own root. + // Abandoning A must only ever reach containers carrying A's own `stack-root` label. + const stateA3 = yield* stateFor(rootA); + const stateB3 = yield* stateFor(rootB); + yield* stateA3.save({ + id: stackId, + runtime: "docker", + identity: { + projectRoot: `${base}/project-a`, + branchContext: "container-shutdown-test", + stackName: stackId, + }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + ports: [], + }); + yield* stateB3.save({ + id: stackId, + runtime: "docker", + identity: { + projectRoot: `${base}/project-b`, + branchContext: "container-shutdown-test", + stackName: stackId, + }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + ports: [], + }); + const accessA3 = yield* launchHost(stateA3, { + stateRoot: rootA, + cacheRoot, + stackId, + entrypoint: shortRegistrationPollFixture, + }); + activeA = accessA3.endpoint; + stoppedA = false; + const accessB3 = yield* launchHost(stateB3, { stateRoot: rootB, cacheRoot, stackId }); + activeB = accessB3.endpoint; + stoppedB = false; + + const clientA3 = yield* ownerClient(accessA3); + const clientB3 = yield* ownerClient(accessB3); + const mailA3 = yield* clientA3.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* clientA3.startService({ id: mailA3.id }); + yield* clientA3.readyService({ id: mailA3.id }); + const mailB3 = yield* clientB3.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* clientB3.startService({ id: mailB3.id }); + yield* clientB3.readyService({ id: mailB3.id }); + + expect( + (yield* containers(stackId, dataA)).length, + "A runs its mail container", + ).toBeGreaterThan(0); + const idsB3 = yield* containers(stackId, dataB); + expect(idsB3.length, "B runs its mail container").toBeGreaterThan(0); + + const leaseReleasedA3 = yield* watchLeaseRelease(rootA, stackId); + yield* fs.remove(`${rootA}/${stackId}/state.json`); + yield* leaseReleasedA3; + yield* waitForOwnerExit(accessA3.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.timeout("30 seconds"), + ); + stoppedA = true; + activeA = undefined; + + expect( + yield* containers(stackId, dataA), + "abandoning A removes only A's containers", + ).toEqual([]); + expect( + yield* containers(stackId, dataB), + "abandoning A leaves B's containers running", + ).toEqual(idsB3); + // B's own container still answers a real readiness probe, proving abandonment never + // touched it. + yield* clientB3.readyService({ id: mailB3.id }); + + yield* shutdownOwner(accessB3, true); + yield* waitForOwnerExit(accessB3.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.timeout("15 seconds"), + ); + stoppedB = true; + activeB = undefined; + expect(yield* containers(stackId, dataB), "destroy removes B containers").toEqual([]); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 180_000 }, ); it.live.skipIf(process.platform === "win32")( - "removes its containers and exits when its registration is confirmed gone (F6)", + "removes its containers and exits when its registration is confirmed gone", () => Effect.scoped( Effect.gen(function* () { @@ -272,7 +372,7 @@ it.live.skipIf(process.platform === "win32")( ports: [], }); yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); - // The shortened poll interval (F6) comes only from this dedicated test entrypoint, through + // The shortened poll interval comes only from this dedicated test entrypoint, through // the internal `Context.Reference`; production startup never reads an env var or `Config`. const access = yield* launchHost(state, { stateRoot, @@ -304,7 +404,7 @@ it.live.skipIf(process.platform === "win32")( ).toBe(true); // Subscribes to the owner's own exit signal (its lease release) before triggering the - // deletion, rather than polling for it afterward (F9). + // deletion, rather than polling for it afterward. const leaseReleased = yield* watchLeaseRelease(stateRoot, stackId); yield* fs.remove(`${stateRoot}/${stackId}/state.json`); yield* leaseReleased; @@ -319,15 +419,123 @@ it.live.skipIf(process.platform === "win32")( ).toBe(false); expect( yield* fs.exists(containerEnvRoot), - "abandonment removes the shared container-env scratch directory (F7)", + "abandonment removes the shared container-env scratch directory", + ).toBe(false); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 180_000 }, +); + +it.live.skipIf(process.platform === "win32")( + "abandons a stack whose registration disappears while a signal-driven stop is still draining", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-abandon-gated-" }); + const stackId = `abandon-gated-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const dataRoot = yield* makeDockerDatabaseRoot("stack-abandon-gated-data-", stackId).pipe( + Effect.flatMap(fs.realPath), + ); + const stateRoot = path.dirname(path.dirname(dataRoot)); + const cacheRoot = `${base}/cache`; + const gateDir = `${base}/gate`; + yield* fs.makeDirectory(gateDir, { recursive: true }); + const state = yield* stateFor(stateRoot); + yield* state.save({ + id: stackId, + runtime: "docker", + identity: { + projectRoot: `${base}/project`, + branchContext: "abandon-gated-test", + stackName: stackId, + }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + ports: [], + }); + yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); + const access = yield* launchHost(state, { + stateRoot, + cacheRoot, + stackId, + entrypoint: gatedDrainFixture, + entrypointArgs: [gateDir], + }); + const client = yield* ownerClient(access); + const database = yield* client.createService({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("abandon-gated-password"), + jwtSecret: Redacted.make("abandon-gated-jwt-secret-at-least-thirty-two-characters"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* client.startService({ id: database.id }); + yield* client.readyService({ id: database.id }); + const status = yield* client.status({ id: database.id }); + const port = status.endpoints.find((endpoint) => endpoint.name === "sql")?.port; + if (port === undefined) return yield* Effect.die("Missing database sql endpoint"); + const containerEnvRoot = `${dataRoot}/.container-env`; + expect( + yield* fs.exists(containerEnvRoot), + "the stack's shared container-env scratch directory exists", + ).toBe(true); + + // Keeps the sql listener's outstanding-connection count above zero for the whole gated + // window, so drain can only resolve through the deadline this test controls, never + // through every connection reaching zero on its own. + const held = yield* Effect.acquireRelease( + Effect.callback((resume) => { + const connection = Net.createConnection({ host: "127.0.0.1", port }); + connection.once("connect", () => resume(Effect.succeed(connection))); + connection.once("error", (cause) => resume(Effect.die(cause))); + return Effect.sync(() => connection.destroy()); + }), + (connection) => Effect.sync(() => connection.destroy()), + ); + + // Subscribes to the drain-deadline wait's own entry marker before sending the signal that + // triggers it, so the registration deletion below never races the gate itself. + const waiting = yield* fs.watch(gateDir).pipe( + Stream.filter((event) => event.path === "waiting"), + Stream.take(1), + Stream.runDrain, + Effect.forkChild({ startImmediately: true }), + ); + process.kill(access.endpoint.pid, "SIGTERM"); + yield* Fiber.join(waiting).pipe(Effect.timeout("30 seconds")); + + yield* fs.remove(`${stateRoot}/${stackId}/state.json`); + yield* fs.writeFileString(`${gateDir}/release`, ""); + yield* Effect.sync(() => held.destroy()); + + yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.retry({ while: hasReason("owner-exit-pending") }), + Effect.timeout("30 seconds"), + ); + + expect( + yield* fs.exists(`${stateRoot}/${stackId}/state.json`), + "no registration is republished", + ).toBe(false); + expect( + yield* fs.exists(containerEnvRoot), + "the registration lost during the gated stop still reaches abandonment's cleanup", ).toBe(false); + expect(yield* containers(stackId, dataRoot), "no containers are left behind").toEqual([]); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 180_000 }, ); it.live.skipIf(process.platform === "win32")( - "removes an orphaned container by label alone, independent of any instance's own cleanup (pass 3, C)", + "removes an orphaned container by label alone, independent of any instance's own cleanup", () => Effect.scoped( Effect.gen(function* () { @@ -365,7 +573,7 @@ it.live.skipIf(process.platform === "win32")( }); // No registered service at all, so the per-instance cleanup loop has nothing to do: this // container carries only the stack's identity label, simulating a leaked storage helper - // no claim or helper-registry bookkeeping ever reaches (F5/pass 3, C). Only the + // no claim or helper-registry bookkeeping ever reaches. Only the // registration-independent label sweep can remove it. yield* docker([ "run", @@ -461,7 +669,7 @@ it.live.skipIf(process.platform === "win32")( const port = status.endpoints.find((endpoint) => endpoint.name === "http")?.port; if (port === undefined) return yield* Effect.die("Missing mail http endpoint"); - // Deletes the actual `` itself (F2), not just the entries inside + // Deletes the actual `` itself, not just the entries inside // `/`: its sibling `.registry-lock.sqlite` (`Ports.ts`'s per-root // reservation registry) is gone too, so cleanup can only come from the owner's in-memory // resources, and must never open that registry at all. The lease file goes with it, so @@ -481,7 +689,7 @@ it.live.skipIf(process.platform === "win32")( "abandonment removes containers from in-memory resources alone", ).toEqual([]); - // The port reservation is never released by abandonment (simplification over F2/F7): + // The port reservation is never released by abandonment: // a fresh stack can still claim the exact same port, through `Ports.ts`'s own lazy // reclamation once the former holder's registration is confirmed gone. const reclaimedBase = yield* fs.makeTempDirectoryScoped({ diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index d97d4b5404..88da4f221e 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -52,7 +52,7 @@ import { makeCommandAttachments } from "./host/CommandAttachments.ts"; import * as CommandRunner from "./host/CommandRunner.ts"; /** - * How often an owner confirms its own registration still exists (F6). Internal only: tests shorten + * How often an owner confirms its own registration still exists. Internal only: tests shorten * it through a dedicated entrypoint that overrides this reference, never through env or `Config`. */ export const RegistrationCheckInterval = Context.Reference( @@ -91,7 +91,7 @@ const hostError = (operation: string, cause: unknown, reason?: "runtime-unavaila * `StackHostError`'s own `reason` has carried since before phase 2d, so `destroy`/`shutdown` * (`effect.ts`) know to leave a stack registered for retry instead of surfacing a hard failure. * Unrelated to abandonment's own cleanup-retry decision, which probes the engine directly instead - * of matching a message at all (pass 3, B). + * of matching a message at all. */ const engineUnreachableAtStartup = (cause: unknown): boolean => { const message = cause instanceof Error ? cause.message : String(cause); @@ -192,7 +192,7 @@ export interface StackHostRuntime { destroy: boolean, response?: ReturnType, ) => Effect.Effect; - /** F6: ends ownership after a confirmed-gone registration; joins an already-running shutdown. */ + /** Ends ownership after a confirmed-gone registration; joins an already-running shutdown. */ readonly abandon: Effect.Effect; readonly exit: Deferred.Deferred; } @@ -255,9 +255,9 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( // Admits `mode`'s cleanup when nothing else is in flight: forks it, tracks it as `current` // until it settles (success or failure alike), then clears `current` so a later claim — - // for example abandon after an in-flight stop or destroy — can be admitted again (pass 3, - // A). Only `stop` and `destroy` can actually fail; their own recovery (reopening admission) - // runs, and `current` is cleared, before `claim` below can ever admit anything else. + // for example abandon after an in-flight stop or destroy — can be admitted again. Only + // `stop` and `destroy` can actually fail; their own recovery (reopening admission) runs, + // and `current` is cleared, before `claim` below can ever admit anything else. const begin = ( mode: ShutdownMode, response?: ReturnType, @@ -294,7 +294,7 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( ? Effect.gen(function* () { // Attachment/command cleanup and namespace cleanup run concurrently; each // retries until confirmed (or the engine is confirmed unavailable), and this - // waits for both before the owner is permitted to exit (pass 3, A). + // waits for both before the owner is permitted to exit. const commandCleanup = settled( "command cleanup", Owner.retryUntilConfirmed( @@ -371,7 +371,7 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( mode !== "abandon" && !retiringAfterDestroyFailure ? owner .setDraining(false) - // F6: drain may already have run (and so left the network's own admission + // Drain may already have run (and so left the network's own admission // closed) before this later stage failed; recovering it here keeps a // start/restart that follows from refusing as still draining. .pipe( @@ -387,7 +387,7 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( return fiber; }); - // The one shutdown pipeline (pass 3, A): `stop`, `destroy` and `abandon` are claimed + // The one shutdown pipeline: `stop`, `destroy` and `abandon` are claimed // through the same gate and `current` record, never a separate join. If a shutdown is // already in flight when abandonment is requested, this waits for it to settle (whatever // its outcome) and then claims abandon fresh, unless that settlement was a confirmed @@ -450,7 +450,7 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( claim(destroy ? "destroy" : "stop", response).pipe( Effect.mapError((cause) => stackError("shutdown", cause)), ); - // F6: a registration-loss poll drives this, never an RPC caller, so there is no response to + // A registration-loss poll drives this, never an RPC caller, so there is no response to // watch and no registration left to touch; `claim` never actually fails for this mode. const abandon = claim("abandon").pipe(Effect.ignore); const handlers = StackRpc.of({ @@ -517,9 +517,28 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( type HostEvent = "SIGTERM" | "SIGINT" | "creator-gone" | "registration-gone"; /** - * Polls for a confirmed-gone registration (F6) and offers `registration-gone` once. Any other - * error (permissions, an unmounted root mid-read) keeps the owner running; only a confirmed ENOENT - * (`state.read` returning `undefined`) counts as abandonment. + * True only on a confirmed ENOENT (`state.read` returning `undefined`); any other error + * (permissions, an unmounted root mid-read) keeps the owner running, so it reads as "not + * confirmed gone" rather than abandonment. + */ +const registrationConfirmedGone = Effect.fn("StackHost.registrationConfirmedGone")(function* ( + state: StackNamespace.Interface, + id: string, +) { + const saved = yield* state.read(id).pipe( + Effect.tapError((cause) => + Effect.logWarning( + "Could not confirm the stack registration; keeping the owner running", + cause, + ), + ), + Effect.option, + ); + return Option.isSome(saved) && saved.value === undefined; +}); + +/** + * Polls for a confirmed-gone registration and offers `registration-gone` once. */ const pollRegistration = Effect.fn("StackHost.pollRegistration")(function* ( state: StackNamespace.Interface, @@ -529,16 +548,7 @@ const pollRegistration = Effect.fn("StackHost.pollRegistration")(function* ( const interval = yield* RegistrationCheckInterval; while (true) { yield* Effect.sleep(interval); - const saved = yield* state.read(id).pipe( - Effect.tapError((cause) => - Effect.logWarning( - "Could not confirm the stack registration; keeping the owner running", - cause, - ), - ), - Effect.option, - ); - if (Option.isSome(saved) && saved.value === undefined) { + if (yield* registrationConfirmedGone(state, id)) { yield* Queue.offer(events, "registration-gone"); return; } @@ -733,7 +743,12 @@ export const runStackHost = Effect.fn("StackHost.run")( Effect.tapCause((cause) => Effect.logError("Stack shutdown failed", cause)), Effect.matchCause({ onSuccess: () => true, onFailure: () => false }), ); - if (shutdownSucceeded) break; + if (!shutdownSucceeded) continue; + // A registration deletion that arrives while this data-preserving stop is already + // running only reaches `events` as a queued, now-unread `registration-gone`: a direct + // re-read through the same state API the poll uses catches it before the owner exits. + if (yield* registrationConfirmedGone(state, id)) yield* started.abandon; + break; } }), ).pipe( diff --git a/packages/stack/src/lifetime.integration.test.ts b/packages/stack/src/lifetime.integration.test.ts index e37b7e7145..6643b12ec3 100644 --- a/packages/stack/src/lifetime.integration.test.ts +++ b/packages/stack/src/lifetime.integration.test.ts @@ -248,7 +248,7 @@ it.live.skipIf(process.platform === "win32")( composition: { members: [], dependencies: [] }, ports: [], }); - // The shortened poll interval (F6) comes only from this dedicated test entrypoint, through + // The shortened poll interval comes only from this dedicated test entrypoint, through // the internal `Context.Reference`; production startup never reads an env var or `Config`. const access = yield* launchHost(state, { stateRoot, @@ -272,7 +272,7 @@ it.live.skipIf(process.platform === "win32")( expect(owned.length, "the owner runs the native mail service").toBeGreaterThan(0); // Subscribes to the owner's own exit signal (its lease release, the owner's last act) before - // triggering the deletion, rather than polling for it afterward (F9). + // triggering the deletion, rather than polling for it afterward. const leaseReleased = yield* watchLeaseRelease(stateRoot, stackId); yield* fs.remove(`${stateRoot}/${stackId}/state.json`); yield* leaseReleased; diff --git a/packages/stack/src/runtime/Container.integration.test.ts b/packages/stack/src/runtime/Container.integration.test.ts index 11702702ff..4d390566ba 100644 --- a/packages/stack/src/runtime/Container.integration.test.ts +++ b/packages/stack/src/runtime/Container.integration.test.ts @@ -1495,7 +1495,7 @@ describe("container process adapter", () => { ), ); - it.live("probes a real connection refusal to the engine as unreachable (pass 3, B)", () => + it.live("probes a real connection refusal to the engine as unreachable", () => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; // A real connection attempt, not a mocked one: nothing listens on this loopback port, so @@ -1512,11 +1512,11 @@ describe("container process adapter", () => { }).pipe(Effect.provide(NodeServices.layer)), ); - it.live("probes a real missing engine binary as unreachable (pass 3, B)", () => + it.live("probes a real missing engine binary as unreachable", () => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const fs = yield* FileSystem.FileSystem; - // A directory deliberately containing no `docker` binary, real `PATH` lookup included: the + // A directory containing no `docker` binary, real `PATH` lookup included: the // probe observes the engine's own spawn failure directly, never a stubbed spawner. const emptyBinDir = yield* fs.makeTempDirectoryScoped({ prefix: "engine-missing-bin-" }); // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the spawned child inherits PATH; this is not application config. @@ -1541,16 +1541,21 @@ describe("container process adapter", () => { ); it.live( - "keeps retrying a daemon-relayed failure, such as a registry pull refusal, while the real engine stays reachable (pass 3, B)", + "keeps retrying a daemon-relayed failure, such as a registry pull refusal, while the real engine stays reachable", () => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const target = yield* resolveEngineTarget(spawner); const attempts = yield* Ref.make(0); + const thirdAttemptObserved = yield* Deferred.make(); // Simulates a registry rejecting a pull while the daemon itself answers normally: this - // must never be mistaken for an unavailable engine (F4), so `retryUntilConfirmed` keeps - // retrying for as long as the probe keeps confirming the daemon is reachable. + // must never be mistaken for an unavailable engine, so `retryUntilConfirmed` keeps + // retrying for as long as the probe keeps confirming the daemon is reachable. A third + // attempt is this test's own observed signal, so a slow individual probe cannot flake it. const daemonRelayedFailure = Ref.updateAndGet(attempts, (count) => count + 1).pipe( + Effect.tap((count) => + count >= 3 ? Deferred.succeed(thirdAttemptObserved, undefined) : Effect.void, + ), Effect.andThen( Effect.fail( new ContainerError({ @@ -1561,14 +1566,18 @@ describe("container process adapter", () => { ), ), ); - const outcome = yield* Owner.retryUntilConfirmed( - daemonRelayedFailure, - probeEngineReachable(spawner, target), - ).pipe(Effect.timeoutOption("1.5 seconds")); - expect(Option.isNone(outcome), "never confirmed, and the engine probe never gave up").toBe( - true, + // Backoff between attempts is this test's only controlled time; the daemon probe + // itself stays real, so the attempts it takes to observe a third one are never rushed. + const retrying = yield* Effect.forkDetach( + Owner.retryUntilConfirmed( + daemonRelayedFailure, + probeEngineReachable(spawner, target), + ).pipe(Effect.provideService(Owner.RetryUntilConfirmedInitialDelay, "1 millis")), ); - expect(yield* Ref.get(attempts)).toBeGreaterThan(1); + // A guard against a genuine hang, not a budget this test's attempt count is sampled from. + yield* Deferred.await(thirdAttemptObserved).pipe(Effect.timeout("30 seconds")); + yield* Fiber.interrupt(retrying); + expect(yield* Ref.get(attempts)).toBeGreaterThanOrEqual(3); }).pipe(Effect.provide(NodeServices.layer)), ); diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index dfb51e56a6..1a98876d9b 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -200,7 +200,7 @@ const retryablePull = (error: ContainerError) => rateLimited(error) || transient /** * Probes the pinned engine target directly, with the cheapest real call (`info`), rather than - * classifying a failure's message (pass 3, B): a spawn failure (missing binary) or the daemon + * classifying a failure's message: a spawn failure (missing binary) or the daemon * endpoint refusing the connection both resolve `false`; any other failure (a daemon-relayed * error, for example a registry rejecting a pull while the daemon itself is up) never reaches * this probe and so never resolves `false` through it. @@ -638,10 +638,10 @@ export const makeContainerRuntime = (options: { .pipe(Effect.mapError((cause) => errorFor("environment", cause))); // Not `makeTempDirectoryScoped`: its scoped cleanup removes without `force`, then dies on // any failure, including a confirmed ENOENT when this stack's whole root (this directory's - // owned ancestor) is already gone — turning an ordinary stop into an unrecoverable defect - // (F5/F1 root cause). `acquireRelease` keeps the plain `makeTempDirectory` and installing its - // tolerant, `force: true` finalizer uninterruptible together (F8), so an interruption between - // the two can never leave the directory created but unregistered for cleanup. + // owned ancestor) is already gone — turning an ordinary stop into an unrecoverable defect. + // `acquireRelease` keeps the plain `makeTempDirectory` and installing its tolerant, + // `force: true` finalizer uninterruptible together, so an interruption between the two + // can never leave the directory created but unregistered for cleanup. const directory = yield* Effect.acquireRelease( fs .makeTempDirectory({ directory: containerEnvRoot, prefix: "container-" }) @@ -1059,13 +1059,16 @@ export const removeContainerById = Effect.fn("Container.removeContainerById")( ); /** - * Lists every container carrying this stack's identity label, through the pinned engine: - * service containers and every storage helper alike (per-instance and shared), independent of - * any in-memory registry's own bookkeeping (pass 3, C). + * Lists every container carrying this stack's identity and data-root labels, through the pinned + * engine: service containers and every storage helper alike (per-instance and shared), + * independent of any in-memory registry's own bookkeeping. The data-root label is required + * alongside the stack id label because a stack id alone does not identify a stack: two owners can + * share one id while rooted at different data directories. */ const listStackContainers = Effect.fn("Container.listStackContainers")(function* (options: { readonly target: EngineTarget; readonly stackId: string; + readonly stackRoot: string; }) { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const output = yield* runRaw(spawner, [ @@ -1076,6 +1079,8 @@ const listStackContainers = Effect.fn("Container.listStackContainers")(function* "--no-trunc", "--filter", `label=com.supabase.stack=${options.stackId}`, + "--filter", + `label=com.supabase.stack-root=${options.stackRoot}`, ]); return output .split("\n") @@ -1084,13 +1089,17 @@ const listStackContainers = Effect.fn("Container.listStackContainers")(function* }); /** - * Removes every container carrying this stack's identity label and reports how many remain - * (normally 0): the registration-independent confirming sweep abandonment uses instead of - * depending on any helper registry's own bookkeeping (pass 3, C). A container that disappears + * Removes every container carrying this stack's identity and data-root labels and reports how + * many remain (normally 0): the registration-independent confirming sweep abandonment uses + * instead of depending on any helper registry's own bookkeeping. A container that disappears * between listing and removal is not an error, matching {@link removeContainerById}. */ export const removeStackContainers = Effect.fn("Container.removeStackContainers")( - function* (options: { readonly target: EngineTarget; readonly stackId: string }) { + function* (options: { + readonly target: EngineTarget; + readonly stackId: string; + readonly stackRoot: string; + }) { const ids = yield* listStackContainers(options); yield* Effect.forEach(ids, (id) => removeContainerById({ target: options.target, id }), { concurrency: "unbounded", diff --git a/packages/stack/src/services/Functions.integration.test.ts b/packages/stack/src/services/Functions.integration.test.ts index 080624a3a3..e89e07d3d9 100644 --- a/packages/stack/src/services/Functions.integration.test.ts +++ b/packages/stack/src/services/Functions.integration.test.ts @@ -187,9 +187,9 @@ describe("service catalog", () => { const outside = yield* fs.makeTempDirectoryScoped({ prefix: "functions-symlink-outside-", }); - // The pre-F2 removeData reached for `/functions` specifically, not the whole - // runtime directory; the sentinel sits exactly there so this test actually proves that - // previously endangered data survives, not just the symlink target's top level. + // removeData reaches for `/functions` specifically, not the whole runtime + // directory; the sentinel sits there so this test proves that previously endangered + // data survives, not just the symlink target's top level. const sentinel = path.join(outside, "functions", "sentinel"); yield* fs.makeDirectory(path.join(outside, "functions"), { recursive: true }); yield* fs.writeFileString(sentinel, "do not remove me"); diff --git a/packages/stack/tests/gated-drain-fixture.ts b/packages/stack/tests/gated-drain-fixture.ts new file mode 100644 index 0000000000..4943fd13a9 --- /dev/null +++ b/packages/stack/tests/gated-drain-fixture.ts @@ -0,0 +1,114 @@ +import { Cause, Effect, Exit, Fiber, FileSystem, Option, Path, Schema, Stream } from "effect"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- readiness is an inherited launcher descriptor. +import { closeSync, writeSync } from "node:fs"; +import * as Network from "../src/Network.ts"; +import { runStackHost, StackHostError } from "../src/StackHost.ts"; + +/** + * A real owner entrypoint, identical to `internal/host-process.ts`, except it overrides + * `Network.ShutdownDrainDeadline` to wait for a `release` file in `gateDir` instead of racing a + * real timer, so a test can hold a shutdown's drain open deterministically from outside the + * subprocess. Production code never reads this from an environment variable or `Config`; only + * this dedicated test fixture supplies a different value. + */ +const awaitRelease = (gateDir: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const target = path.join(gateDir, "release"); + // Subscribes before signaling "waiting" below, so a `release` written the instant a test + // observes that signal can never land in the gap between the signal and this watch attaching. + const released = yield* fs.watch(gateDir).pipe( + Stream.filter((event) => event.path === "release" || event.path === target), + Stream.take(1), + Stream.runDrain, + Effect.forkChild({ startImmediately: true }), + ); + // Marks entry into the drain-deadline wait itself, so a test can subscribe to this instead + // of a sleep before it is safe to act on the assumption that drain has actually begun. + yield* fs.writeFileString(path.join(gateDir, "waiting"), ""); + if (yield* fs.exists(target)) return; + yield* Fiber.join(released); + }); + +const writeLine = (value: unknown) => + Effect.gen(function* () { + const serialized = yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))( + value, + ).pipe( + Effect.mapError( + (cause) => new StackHostError({ operation: "ready", message: String(cause), cause }), + ), + ); + yield* Effect.try({ + try: () => writeSync(3, Buffer.from(`${serialized}\n`, "utf8")), + catch: (cause) => new StackHostError({ operation: "ready", message: String(cause), cause }), + }); + }).pipe( + Effect.ensuring( + Effect.sync(() => { + try { + closeSync(3); + } catch { + // The parent may close the inherited descriptor during cancellation. + } + }), + ), + ); + +const [stateRoot, cacheRoot, stackId, gateDir, ...rest] = process.argv.slice(2); + +const program = Effect.gen(function* () { + if ( + stateRoot === undefined || + cacheRoot === undefined || + stackId === undefined || + gateDir === undefined || + rest.length > 0 + ) + return yield* new StackHostError({ + operation: "startup", + message: "Expected stateRoot, cacheRoot, stackId and gateDir", + }); + let reported = false; + const report = (value: unknown) => + Effect.suspend(() => { + if (reported) return Effect.void; + reported = true; + return writeLine(value); + }); + yield* runStackHost({ + stateRoot, + cacheRoot, + stackId, + onReady: ({ endpoint, secret }) => report({ type: "ready", endpoint, secret }), + }).pipe( + Effect.provideService(Network.ShutdownDrainDeadline, awaitRelease(gateDir)), + Effect.catchCause((cause) => { + const failure = Option.getOrUndefined(Cause.findErrorOption(cause)); + return report({ + type: "error", + message: failure?.message ?? Cause.pretty(cause), + ...(failure?.reason === undefined ? {} : { reason: failure.reason }), + }).pipe(Effect.exit, Effect.andThen(Effect.failCause(cause))); + }), + ); +}); + +const flushed = (stream: NodeJS.WriteStream) => + Effect.callback((resume) => { + stream.write("", () => resume(Effect.void)); + }).pipe(Effect.timeoutOption("1 second")); + +await Effect.runPromise( + program.pipe( + Effect.exit, + Effect.tap((exit) => + Exit.isFailure(exit) + ? Effect.sync(() => process.stderr.write(`${Cause.pretty(exit.cause)}\n`)) + : Effect.void, + ), + Effect.tap(() => Effect.all([flushed(process.stdout), flushed(process.stderr)])), + Effect.flatMap((exit) => Effect.sync(() => process.exit(Exit.isSuccess(exit) ? 0 : 1))), + ), +); diff --git a/packages/stack/tests/short-registration-poll-fixture.ts b/packages/stack/tests/short-registration-poll-fixture.ts index 2ea68593fb..c6e2cf924c 100644 --- a/packages/stack/tests/short-registration-poll-fixture.ts +++ b/packages/stack/tests/short-registration-poll-fixture.ts @@ -6,7 +6,7 @@ import { RegistrationCheckInterval, runStackHost, StackHostError } from "../src/ /** * A real owner entrypoint, identical to `internal/host-process.ts`, except it overrides the - * registration-loss poll interval (F6) through the sanctioned internal `Context.Reference` so an + * registration-loss poll interval through the sanctioned internal `Context.Reference` so an * abandonment test does not wait a real 30 seconds. Production code never reads this from an * environment variable or `Config`; only this dedicated test fixture supplies a different value. */ From f1f4cb734dcf49d2040b689921e4a7fdc7742ae7 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 13:15:11 +0200 Subject: [PATCH 039/142] test(stack): cover gateway drain through a real owner shutdown --- packages/stack/src/Network.ts | 5 +- packages/stack/src/Service.ts | 6 +- ...ost.container-shutdown.integration.test.ts | 296 +++++++++++++ .../stack/src/StackHost.integration.test.ts | 414 +++++++----------- 4 files changed, 446 insertions(+), 275 deletions(-) diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index 8cd745f1fe..274a218944 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -21,10 +21,7 @@ export type NetworkRuntime = "native" | "docker"; /** * The shutdown-drain deadline: established connections keep flowing until this event - * completes. An injectable event rather than a duration, so a test can gate when it fires - * through a `Deferred` instead of racing a shortened real timer; production always - * defaults to a plain 10-second sleep. Internal only, following the `RegistrationCheckInterval` - * pattern: production never reads it from an environment variable or `Config`. + * completes. An event rather than a duration so a test can gate exactly when it fires. */ export const ShutdownDrainDeadline = Context.Reference>( "@supabase/stack/ShutdownDrainDeadline", diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index 0f37ee868d..7cc957d96a 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -492,10 +492,8 @@ export const makeService = ( const leftover = yield* Ref.get(current); if (leftover !== undefined) { yield* stopRecord(leftover, true); - // A previous stop attempt may have failed and reported `StopFailed`, leaving the - // reducer in `Stopping` forever; this retry's own confirmed termination must reach it - // too, the same way `stop`'s own successful path does, or the reducer (and anything - // gated on it, such as `NetworkNamespace.release`'s endpoint check) never learns. + // A previous failed stop can leave the reducer in `Stopping` forever; report this + // retry's termination too, or `NetworkNamespace.release`'s endpoint check never learns. yield* report( LifecycleEvent.Exited({ id, diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index a91bc68261..ee9079d3a0 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -1,9 +1,12 @@ +import { PgClient } from "@effect/sql-pg"; import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; import { Context, Crypto, + Data, Effect, + Exit, FileSystem, Fiber, Layer, @@ -12,6 +15,7 @@ import { Schedule, Stream, } from "effect"; +import * as Reactivity from "effect/unstable/reactivity/Reactivity"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import * as Net from "node:net"; // oxlint-disable-line effecttsgo/node-builtin-import -- forces an outstanding connection during a gated drain. import { fileURLToPath } from "node:url"; @@ -35,6 +39,96 @@ const gatedDrainFixture = fileURLToPath( new URL("../tests/gated-drain-fixture.ts", import.meta.url), ); +class HostTestError extends Data.TaggedError("HostTestError")<{ readonly message: string }> {} + +const hostTestError = (cause: unknown) => + new HostTestError({ message: cause instanceof Error ? cause.message : String(cause) }); + +const isHttpRefused = (port: number) => + Effect.callback((resume) => { + let socket: Net.Socket; + try { + socket = Net.connect(port, "127.0.0.1"); + } catch { + resume(Effect.succeed(true)); + return Effect.void; + } + let response = ""; + const onConnect = () => + socket.write("GET / HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n"); + const onData = (bytes: Buffer) => (response += bytes.toString()); + const settle = () => resume(Effect.succeed(response.length === 0)); + socket.once("connect", onConnect); + socket.on("data", onData); + socket.once("close", settle); + socket.once("error", settle); + return Effect.sync(() => { + socket.off("connect", onConnect); + socket.off("data", onData); + socket.off("close", settle); + socket.off("error", settle); + socket.destroy(); + }); + }); + +/** + * Opens a real TCP connection to the mail service's send endpoint and writes only the first half + * of the request body, so the request stays genuinely in flight until `complete` sends the rest. + */ +const openPartialMailSend = (port: number, subject: string) => + Effect.gen(function* () { + // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- raw wire payload split mid-body to hold the request open, not a domain model + const payload = JSON.stringify({ + From: { Email: "sender@example.com" }, + To: [{ Email: "recipient@example.com" }], + Subject: subject, + Text: "drain test", + }); + const splitAt = Math.ceil(payload.length / 2); + const socket = yield* Effect.acquireRelease( + Effect.callback((resume) => { + let socket: Net.Socket; + try { + socket = Net.connect(port, "127.0.0.1"); + } catch (cause) { + resume(Effect.fail(hostTestError(cause))); + return Effect.void; + } + const onConnect = () => + socket.write( + `POST /api/v1/send HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Type: application/json\r\nContent-Length: ${payload.length}\r\nConnection: close\r\n\r\n${payload.slice(0, splitAt)}`, + () => resume(Effect.succeed(socket)), + ); + const onError = (cause: Error) => resume(Effect.fail(hostTestError(cause))); + socket.once("connect", onConnect); + socket.once("error", onError); + return Effect.sync(() => { + socket.off("connect", onConnect); + socket.off("error", onError); + }); + }), + (socket) => Effect.sync(() => socket.destroy()), + ); + const response = yield* Effect.callback((resume) => { + let buffer = ""; + const onData = (bytes: Buffer) => (buffer += bytes.toString()); + const onEnd = () => resume(Effect.succeed(buffer)); + const onError = (cause: Error) => resume(Effect.fail(hostTestError(cause))); + socket.on("data", onData); + socket.once("end", onEnd); + socket.once("error", onError); + return Effect.sync(() => { + socket.off("data", onData); + socket.off("end", onEnd); + socket.off("error", onError); + }); + }).pipe(Effect.forkChild({ startImmediately: true })); + return { + complete: () => socket.write(payload.slice(splitAt)), + response: Fiber.join(response), + }; + }); + const helperImage = "public.ecr.aws/docker/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; @@ -730,3 +824,205 @@ it.live.skipIf(process.platform === "win32")( ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 180_000 }, ); + +it.live.skipIf(process.platform === "win32")( + "drains an in-flight mail request through a real docker owner shutdown, and refuses a new connection once draining begins", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-drain-docker-" }); + const stackId = `drain-docker-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const dataRoot = yield* makeDockerDatabaseRoot("stack-drain-docker-data-", stackId).pipe( + Effect.flatMap(fs.realPath), + ); + const stateRoot = path.dirname(path.dirname(dataRoot)); + const cacheRoot = `${base}/cache`; + const gateDir = `${base}/gate`; + yield* fs.makeDirectory(gateDir, { recursive: true }); + const state = yield* stateFor(stateRoot); + yield* state.save({ + id: stackId, + runtime: "docker", + identity: { + projectRoot: `${base}/project`, + branchContext: "drain-docker-test", + stackName: stackId, + }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + ports: [], + }); + yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); + const access = yield* launchHost(state, { + stateRoot, + cacheRoot, + stackId, + entrypoint: gatedDrainFixture, + entrypointArgs: [gateDir], + }); + const client = yield* ownerClient(access); + const mail = yield* client.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* client.startService({ id: mail.id }); + yield* client.readyService({ id: mail.id }); + const status = yield* client.status({ id: mail.id }); + const port = status.endpoints.find((endpoint) => endpoint.name === "http")?.port; + if (port === undefined) return yield* Effect.die("Missing mail http endpoint"); + + const inFlight = yield* openPartialMailSend(port, "drain-docker-inflight"); + + const waiting = yield* fs.watch(gateDir).pipe( + Stream.filter((event) => event.path === "waiting"), + Stream.take(1), + Stream.runDrain, + Effect.forkChild({ startImmediately: true }), + ); + process.kill(access.endpoint.pid, "SIGTERM"); + yield* Fiber.join(waiting).pipe(Effect.timeout("30 seconds")); + + expect(yield* isHttpRefused(port), "a new connection is refused once draining begins").toBe( + true, + ); + + inFlight.complete(); + const response = yield* inFlight.response.pipe(Effect.timeout("10 seconds")); + expect( + response.startsWith("HTTP/1.1 200"), + "the in-flight request still completes during drain", + ).toBe(true); + + yield* fs.writeFileString(`${gateDir}/release`, ""); + yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.retry({ while: hasReason("owner-exit-pending") }), + Effect.timeout("30 seconds"), + ); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 180_000 }, +); + +it.live.skipIf(process.platform === "win32")( + "keeps a pinned postgres connection serving through a real owner shutdown, and cuts it once the deadline fires", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-drain-postgres-" }); + const stackId = `drain-postgres-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const dataRoot = yield* makeDockerDatabaseRoot("stack-drain-postgres-data-", stackId).pipe( + Effect.flatMap(fs.realPath), + ); + const stateRoot = path.dirname(path.dirname(dataRoot)); + const cacheRoot = `${base}/cache`; + const gateDir = `${base}/gate`; + yield* fs.makeDirectory(gateDir, { recursive: true }); + const state = yield* stateFor(stateRoot); + yield* state.save({ + id: stackId, + runtime: "docker", + identity: { + projectRoot: `${base}/project`, + branchContext: "drain-postgres-test", + stackName: stackId, + }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + ports: [], + }); + yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); + const access = yield* launchHost(state, { + stateRoot, + cacheRoot, + stackId, + entrypoint: gatedDrainFixture, + entrypointArgs: [gateDir], + }); + const client = yield* ownerClient(access); + const password = Redacted.make("drain-postgres-password"); + const database = yield* client.createService({ + service: "database", + config: { + version: "17", + databasePassword: password, + jwtSecret: Redacted.make("drain-postgres-jwt-secret-at-least-thirty-two-characters"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* client.startService({ id: database.id }); + yield* client.readyService({ id: database.id }); + const status = yield* client.status({ id: database.id }); + const port = status.endpoints.find((endpoint) => endpoint.name === "sql")?.port; + if (port === undefined) return yield* Effect.die("Missing database sql endpoint"); + + const services = yield* Layer.build( + Layer.effect( + PgClient.PgClient, + PgClient.makeClient({ + host: "127.0.0.1", + port, + database: "postgres", + username: "supabase_admin", + password, + }), + ).pipe(Layer.provide(Reactivity.layer)), + ); + const sql = Context.get(services, PgClient.PgClient); + yield* sql.unsafe("SELECT 1"); + + // A second, query-free connection to the same listener: its closure is a push-based + // signal that the deadline's cut already reached the gateway, with no query to race. + const probe = yield* Effect.acquireRelease( + Effect.callback((resume) => { + const connection = Net.createConnection({ host: "127.0.0.1", port }); + connection.once("connect", () => resume(Effect.succeed(connection))); + connection.once("error", (cause) => resume(Effect.die(cause))); + return Effect.sync(() => connection.destroy()); + }), + (connection) => Effect.sync(() => connection.destroy()), + ); + + const waiting = yield* fs.watch(gateDir).pipe( + Stream.filter((event) => event.path === "waiting"), + Stream.take(1), + Stream.runDrain, + Effect.forkChild({ startImmediately: true }), + ); + process.kill(access.endpoint.pid, "SIGTERM"); + yield* Fiber.join(waiting).pipe(Effect.timeout("30 seconds")); + + yield* sql.unsafe("SELECT 1"); + + const closed = yield* Effect.callback((resume) => { + if (probe.destroyed) { + resume(Effect.void); + return Effect.void; + } + const onClose = () => resume(Effect.void); + probe.once("close", onClose); + return Effect.sync(() => probe.off("close", onClose)); + }).pipe(Effect.forkChild({ startImmediately: true })); + yield* fs.writeFileString(`${gateDir}/release`, ""); + yield* Fiber.join(closed).pipe(Effect.timeout("30 seconds")); + + const cut = yield* sql.unsafe("SELECT 1").pipe(Effect.timeout("5 seconds"), Effect.exit); + expect(Exit.isFailure(cut), "the connection is cut once the deadline fires").toBe(true); + + yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.retry({ while: hasReason("owner-exit-pending") }), + Effect.timeout("30 seconds"), + ); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 180_000 }, +); diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts index 36165a3bc6..2dd2bb91a2 100644 --- a/packages/stack/src/StackHost.integration.test.ts +++ b/packages/stack/src/StackHost.integration.test.ts @@ -3,6 +3,7 @@ import { expect, it } from "@effect/vitest"; import { Cause, Context, + Crypto, Data, Deferred, Effect, @@ -12,7 +13,6 @@ import { Layer, Redacted, Ref, - Schedule, Schema, Stream, } from "effect"; @@ -21,9 +21,9 @@ import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"; import * as HttpClient from "effect/unstable/http/HttpClient"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- integration observes exact listener closure. import * as Net from "node:net"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- integration drives a real HTTP keep-alive connection through the public listener. -import * as Http from "node:http"; +import { fileURLToPath } from "node:url"; import { + hasReason, launchHost, ownerAuthorization, ownerClient, @@ -31,7 +31,6 @@ import { waitForOwnerExit, type HostAccess, } from "./HostProcess.ts"; -import * as Network from "./Network.ts"; import * as Owner from "./Owner.ts"; import { OrchestratorError } from "./Orchestrator.ts"; import { CommandEvent, StackError } from "./Rpc.ts"; @@ -44,6 +43,10 @@ import { noContainerClaims } from "../tests/claims.ts"; class HostTestError extends Data.TaggedError("HostTestError")<{ readonly message: string }> {} +const gatedDrainFixture = fileURLToPath( + new URL("../tests/gated-drain-fixture.ts", import.meta.url), +); + const stateFor = (root: string) => Effect.gen(function* () { const context = yield* Layer.build(StackNamespace.layer({ root })); @@ -138,6 +141,91 @@ const awaitClosed = (socket: Net.Socket) => return Effect.sync(() => socket.off("close", onClose)); }); +const isHttpRefused = (port: number) => + Effect.callback((resume) => { + let socket: Net.Socket; + try { + socket = Net.connect(port, "127.0.0.1"); + } catch { + resume(Effect.succeed(true)); + return Effect.void; + } + let response = ""; + const onConnect = () => + socket.write("GET / HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n"); + const onData = (bytes: Buffer) => (response += bytes.toString()); + const settle = () => resume(Effect.succeed(response.length === 0)); + socket.once("connect", onConnect); + socket.on("data", onData); + socket.once("close", settle); + socket.once("error", settle); + return Effect.sync(() => { + socket.off("connect", onConnect); + socket.off("data", onData); + socket.off("close", settle); + socket.off("error", settle); + socket.destroy(); + }); + }); + +/** + * Opens a real TCP connection to the mail service's send endpoint and writes only the first half + * of the request body, so the request stays genuinely in flight until `complete` sends the rest. + */ +const openPartialMailSend = (port: number, subject: string) => + Effect.gen(function* () { + // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- raw wire payload split mid-body to hold the request open, not a domain model + const payload = JSON.stringify({ + From: { Email: "sender@example.com" }, + To: [{ Email: "recipient@example.com" }], + Subject: subject, + Text: "drain test", + }); + const splitAt = Math.ceil(payload.length / 2); + const socket = yield* Effect.acquireRelease( + Effect.callback((resume) => { + let socket: Net.Socket; + try { + socket = Net.connect(port, "127.0.0.1"); + } catch (cause) { + resume(Effect.fail(hostTestError(cause))); + return Effect.void; + } + const onConnect = () => + socket.write( + `POST /api/v1/send HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Type: application/json\r\nContent-Length: ${payload.length}\r\nConnection: close\r\n\r\n${payload.slice(0, splitAt)}`, + () => resume(Effect.succeed(socket)), + ); + const onError = (cause: Error) => resume(Effect.fail(hostTestError(cause))); + socket.once("connect", onConnect); + socket.once("error", onError); + return Effect.sync(() => { + socket.off("connect", onConnect); + socket.off("error", onError); + }); + }), + (socket) => Effect.sync(() => socket.destroy()), + ); + const response = yield* Effect.callback((resume) => { + let buffer = ""; + const onData = (bytes: Buffer) => (buffer += bytes.toString()); + const onEnd = () => resume(Effect.succeed(buffer)); + const onError = (cause: Error) => resume(Effect.fail(hostTestError(cause))); + socket.on("data", onData); + socket.once("end", onEnd); + socket.once("error", onError); + return Effect.sync(() => { + socket.off("data", onData); + socket.off("end", onEnd); + socket.off("error", onError); + }); + }).pipe(Effect.forkChild({ startImmediately: true })); + return { + complete: () => socket.write(payload.slice(splitAt)), + response: Fiber.join(response), + }; + }); + const inProcessRuntime = ( owner: Parameters[0], state: StackNamespace.Interface, @@ -1187,290 +1275,82 @@ it.live("destroys a stack only after an abandoned composition settles", () => ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); -/** A running native mail service, reachable through its public HTTP listener like any client. */ -const withRunningMail = Effect.fn("Test.withRunningMail")(function* (prefix: string) { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix }); - const state = yield* stateFor(`${root}/state`); - const saved = { - id: "stack", - runtime: "native" as const, - identity: { projectRoot: root, branchContext: "main", stackName: prefix }, - instances: [], - lifetime: "detached" as const, - composition: { members: [], dependencies: [] }, - ports: [], - }; - yield* state.save(saved); - const owner = yield* ownerFor({ - saved, - state, - root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", - }); - const { runtime } = yield* inProcessRuntime(owner, state, root); - const client = yield* ownerClient(runtime.access); - const mail = yield* client.createService({ - service: "mail", - config: {}, - endpoints: { http: { port: "auto" } }, - }); - yield* client.startService({ id: mail.id }); - yield* client.readyService({ id: mail.id }); - const status = yield* client.status({ id: mail.id }); - const port = status.endpoints.find((endpoint) => endpoint.name === "http")?.port; - if (port === undefined) return yield* Effect.die("Missing mail http endpoint"); - return { runtime, port }; -}); - -/** - * One real HTTP round trip to the public listener over `agent`, proving application data actually - * flows (not just a raw established socket). Resolves once the response ends, with the exact - * socket that carried it, so a caller can later confirm that same connection's fate. - */ -const httpRoundTrip = (port: number, agent: Http.Agent) => - Effect.callback< - { readonly status: number; readonly socket: Net.Socket; readonly body: string }, - HostTestError - >((resume) => { - const request = Http.request( - { host: "127.0.0.1", port, path: "/", method: "GET", agent }, - (response) => { - // Captured immediately: the agent may detach `response.socket` once the response ends. - const socket = response.socket; - let body = ""; - response.on("data", (chunk: Buffer) => { - body += chunk.toString(); - }); - response.once("end", () => { - if (socket === null) { - resume(Effect.fail(new HostTestError({ message: "Response has no socket" }))); - return; - } - resume(Effect.succeed({ status: response.statusCode ?? 0, socket, body })); - }); - }, - ); - request.once("error", (cause) => resume(Effect.fail(hostTestError(cause)))); - request.end(); - return Effect.sync(() => request.destroy()); - }); - -/** `true` once a fresh one-shot request to `port` is refused (reset) rather than answered. */ -const isHttpRefused = (port: number) => - Effect.acquireRelease( - Effect.sync(() => new Http.Agent({ keepAlive: false })), - (agent) => Effect.sync(() => agent.destroy()), - ).pipe( - Effect.flatMap((agent) => httpRoundTrip(port, agent)), - Effect.as(false), - Effect.orElseSucceed(() => true), - ); - -/** - * Waits for drain's accept-close to take effect by actively re-checking refusal, instead of - * guessing a fixed delay: the condition, once true, holds for the whole multi-second drain window, - * so a short bounded retry observes it reliably without racing a single arbitrary sleep. - */ -const waitUntilRefused = (port: number) => - Effect.scoped(isHttpRefused(port)).pipe( - Effect.filterOrFail( - (refused) => refused, - () => new HostTestError({ message: "New connections are not refused yet" }), - ), - Effect.retry({ - schedule: Schedule.spaced("50 millis").pipe(Schedule.upTo({ duration: "5 seconds" })), - }), - ); - -it.live( - "drains an established connection through a stack listener during stop, and refuses a new one", - () => - Effect.scoped( - Effect.gen(function* () { - const { runtime, port } = yield* withRunningMail("stack-host-drain-inflight-"); - const agent = yield* Effect.acquireRelease( - Effect.sync(() => new Http.Agent({ keepAlive: true, maxSockets: 1 })), - (agent) => Effect.sync(() => agent.destroy()), - ); - const first = yield* httpRoundTrip(port, agent); - // Exactly 200, not merely finite: a proxy-generated 502 (the backend unreachable) is - // just as finite a status and must not pass as "a real response". - expect(first.status, "the first request gets a real response before drain").toBe(200); - const shutdown = yield* Effect.forkScoped(shutdownOwner(runtime.access, false)); - yield* waitUntilRefused(port); - expect(first.socket.destroyed, "the established connection survives the refusal").toBe( - false, - ); - const second = yield* httpRoundTrip(port, agent); - expect( - second.socket, - "the second request reuses the same already-established connection", - ).toBe(first.socket); - expect( - second.status, - "a request on the established connection still completes during drain", - ).toBe(200); - // The client finishes on its own, well before the drain deadline. - agent.destroy(); - yield* Fiber.join(shutdown).pipe( - Effect.timeoutOrElse({ - duration: "5 seconds", - orElse: () => - Effect.fail( - new HostTestError({ message: "Stop did not complete once the connection closed" }), - ), - }), - ); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), - { timeout: 60_000 }, -); - -/** - * Drives `network.drain` directly, against a raw backend this test fully controls (not the real - * mail service), so a response can be gated open exactly across the drain boundary instead of - * merely completing fast. The deadline is an injected event (pass 3, F7): `ShutdownDrainDeadline` - * now holds an `Effect`, so the test supplies `Deferred.await(deadline)` and decides - * precisely when it fires, with no clock (real or virtual) involved at all. - */ it.live( - "drains an in-flight response, refuses new connections, and cuts a hanging one only once the deadline completes (F7)", + "drains an in-flight mail request through a real owner shutdown, and refuses a new connection once draining begins", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-network-drain-" }); - const state = yield* stateFor(`${root}/state`); - const stackId = "drain-network-test"; + const crypto = yield* Crypto.Crypto; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-drain-native-" }); + const stackId = `drain-native-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const stateRoot = `${base}/state`; + const cacheRoot = `${base}/cache`; + const gateDir = `${base}/gate`; + yield* fs.makeDirectory(gateDir, { recursive: true }); + const state = yield* stateFor(stateRoot); yield* state.save({ id: stackId, - runtime: "native" as const, - identity: { projectRoot: root, branchContext: "main", stackName: stackId }, + runtime: "native", + identity: { + projectRoot: `${base}/project`, + branchContext: "drain-native-test", + stackName: stackId, + }, instances: [], - lifetime: "detached" as const, + lifetime: "detached", composition: { members: [], dependencies: [] }, ports: [], }); - const networkContext = yield* Layer.build( - Network.layer({ stackId, runtime: "native" }).pipe( - Layer.provide(Layer.succeed(StackNamespace.Service, state)), - ), - ); - const network = Context.get(networkContext, Network.Service); - - // A backend this test fully controls: each request is held open until its matching slot - // is released, so the first can be proven genuinely in flight across the drain boundary, - // and the second deliberately never completes on its own. - const slotA = { - received: yield* Deferred.make(), - release: yield* Deferred.make(), - }; - const slotB = { - received: yield* Deferred.make(), - release: yield* Deferred.make(), - }; - const slots = [slotA, slotB]; - const nextSlot = yield* Ref.make(0); - const services = yield* Effect.context(); - const server = Http.createServer((_request, response) => { - Effect.runForkWith(services)( - Effect.gen(function* () { - const index = yield* Ref.getAndUpdate(nextSlot, (value) => value + 1); - const slot = slots[index]; - if (slot === undefined) { - response.writeHead(503); - response.end(); - return; - } - yield* Deferred.succeed(slot.received, undefined); - yield* Deferred.await(slot.release); - response.writeHead(200, { "content-type": "text/plain" }); - response.end(`drained-ok-${index}`); - }), - ); + const access = yield* launchHost(state, { + stateRoot, + cacheRoot, + stackId, + entrypoint: gatedDrainFixture, + entrypointArgs: [gateDir], }); - const backendPort = yield* Effect.acquireRelease( - Effect.callback((resume) => { - server.listen(0, "127.0.0.1", () => { - const address = server.address(); - resume( - Effect.succeed(address !== null && typeof address === "object" ? address.port : 0), - ); - }); - }), - () => - Effect.callback((resume) => { - server.close(() => resume(Effect.void)); - }), - ); - const namespace = yield* network.register({ - id: "backend", - endpoints: { - http: { - protocol: "http", - port: "auto", - backend: Effect.succeed({ host: "127.0.0.1", port: backendPort }), - enabled: Effect.succeed(true), - }, - }, + const client = yield* ownerClient(access); + const mail = yield* client.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, }); - const bound = yield* namespace.bind; - const port = bound.find((endpoint) => endpoint.name === "http")?.port; - if (port === undefined) return yield* Effect.die("Missing http endpoint"); + yield* client.startService({ id: mail.id }); + yield* client.readyService({ id: mail.id }); + const status = yield* client.status({ id: mail.id }); + const port = status.endpoints.find((endpoint) => endpoint.name === "http")?.port; + if (port === undefined) return yield* Effect.die("Missing mail http endpoint"); - const agentA = yield* Effect.acquireRelease( - Effect.sync(() => new Http.Agent({ keepAlive: false })), - (agent) => Effect.sync(() => agent.destroy()), - ); - const agentB = yield* Effect.acquireRelease( - Effect.sync(() => new Http.Agent({ keepAlive: false })), - (agent) => Effect.sync(() => agent.destroy()), - ); - // Both connections are established and in flight before drain starts: a connection made - // after accept has already closed would be refused outright, never "hanging". - const requestA = yield* Effect.forkScoped(httpRoundTrip(port, agentA)); - yield* Deferred.await(slotA.received); - const requestB = yield* Effect.forkScoped(httpRoundTrip(port, agentB)); - yield* Deferred.await(slotB.received); + const inFlight = yield* openPartialMailSend(port, "drain-native-inflight"); - const deadline = yield* Deferred.make(); - const drain = yield* Effect.forkScoped( - network.drain.pipe( - Effect.provideService(Network.ShutdownDrainDeadline, Deferred.await(deadline)), - ), + // Subscribes to the drain-deadline wait's own entry marker before sending the signal, so + // the refusal check below never races the gate itself. + const waiting = yield* fs.watch(gateDir).pipe( + Stream.filter((event) => event.path === "waiting"), + Stream.take(1), + Stream.runDrain, + Effect.forkChild({ startImmediately: true }), ); - yield* waitUntilRefused(port); + process.kill(access.endpoint.pid, "SIGTERM"); + yield* Fiber.join(waiting).pipe(Effect.timeout("30 seconds")); - // The in-flight response completes with status 200 and its expected body during drain. - yield* Deferred.succeed(slotA.release, undefined); - const resultA = yield* Fiber.join(requestA); - expect(resultA.status, "the in-flight response completes during drain").toBe(200); - expect(resultA.body, "with its expected body").toBe("drained-ok-0"); + expect(yield* isHttpRefused(port), "a new connection is refused once draining begins").toBe( + true, + ); - // The second connection, deliberately never released, stays open: nothing but the test - // itself decides when the deadline elapses. + inFlight.complete(); + const response = yield* inFlight.response.pipe(Effect.timeout("10 seconds")); expect( - requestB.pollUnsafe(), - "the hanging connection stays open before the deadline completes", - ).toBeUndefined(); + response.startsWith("HTTP/1.1 200"), + "the in-flight request still completes during drain", + ).toBe(true); - yield* Deferred.succeed(deadline, undefined); - yield* Fiber.join(drain).pipe( - Effect.timeoutOrElse({ - duration: "5 seconds", - orElse: () => - Effect.fail( - new HostTestError({ message: "Drain did not complete after the deadline" }), - ), - }), + yield* fs.writeFileString(`${gateDir}/release`, ""); + yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.retry({ while: hasReason("owner-exit-pending") }), + Effect.timeout("30 seconds"), ); - const resultB = yield* Fiber.join(requestB).pipe(Effect.exit); - expect( - Exit.isFailure(resultB), - "the hanging connection is cut once the deadline completes", - ).toBe(true); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), - { timeout: 30_000 }, + { timeout: 60_000 }, ); From 5851a6fa0dd3395a13dc6bc306810a3aa27b7424 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 13:50:39 +0200 Subject: [PATCH 040/142] test(stack): attach file watchers before triggering the awaited change --- .oxlintrc.json | 3 +- .../stack/src/HostProcess.integration.test.ts | 40 +++++++++---------- ...ost.container-shutdown.integration.test.ts | 28 ++++--------- .../stack/src/StackHost.integration.test.ts | 10 ++--- .../stack/src/lifetime.integration.test.ts | 34 ++-------------- packages/stack/tests/gated-drain-fixture.ts | 17 +++----- packages/stack/tests/watch-entry.ts | 38 ++++++++++++++++++ 7 files changed, 78 insertions(+), 92 deletions(-) create mode 100644 packages/stack/tests/watch-entry.ts diff --git a/.oxlintrc.json b/.oxlintrc.json index f026649b47..a2d52a5fc0 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -384,7 +384,8 @@ "packages/stack/tests/host-process-fixture.ts", "packages/stack/tests/failing-owner-fixture.ts", "packages/stack/tests/short-registration-poll-fixture.ts", - "packages/stack/tests/gated-drain-fixture.ts" + "packages/stack/tests/gated-drain-fixture.ts", + "packages/stack/tests/watch-entry.ts" ], "rules": { "no-restricted-imports": [ diff --git a/packages/stack/src/HostProcess.integration.test.ts b/packages/stack/src/HostProcess.integration.test.ts index 23ce267e69..b5ef0c5093 100644 --- a/packages/stack/src/HostProcess.integration.test.ts +++ b/packages/stack/src/HostProcess.integration.test.ts @@ -31,6 +31,7 @@ import { } from "./HostProcess.ts"; import { discover } from "./effect.ts"; import { watchLeaseRelease } from "../tests/owner.ts"; +import { watchEntry } from "../tests/watch-entry.ts"; import * as StackNamespace from "./StackNamespace.ts"; class ProcessTestError extends Data.TaggedError("ProcessTestError")<{ readonly message: string }> {} @@ -227,30 +228,29 @@ const closeServer = (server: Net.Server) => server.close(() => resume(Effect.void)); }); +/** Attaches now; the returned effect awaits the marker and reads its value. Subscribe before + * triggering the fixture that writes the marker, then await the returned effect after. */ const waitForMarker = (marker: string) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - yield* fs.watch(path.dirname(marker)).pipe( - Stream.filter((event) => event.path === path.basename(marker)), - Stream.runHead, - Effect.flatMap( - Option.match({ - onNone: () => Effect.fail(new ProcessTestError({ message: "marker watcher ended" })), - onSome: Effect.succeed, + const ready = yield* watchEntry(path.dirname(marker), path.basename(marker), true); + return yield* Effect.succeed( + ready.pipe( + Effect.flatMap(() => + fs.readFileString(marker).pipe( + Effect.map(Number), + Effect.mapError(() => new ProcessTestError({ message: "marker read failed" })), + ), + ), + Effect.timeoutOrElse({ + duration: "10 seconds", + orElse: () => + Effect.fail(new ProcessTestError({ message: "slow fixture did not start" })), }), ), ); - return yield* fs.readFileString(marker).pipe( - Effect.map(Number), - Effect.mapError(() => new ProcessTestError({ message: "marker read failed" })), - ); - }).pipe( - Effect.timeoutOrElse({ - duration: "10 seconds", - orElse: () => Effect.fail(new ProcessTestError({ message: "slow fixture did not start" })), - }), - ); + }); it.live("starts exactly one owner for concurrent launchers and attaches the others", () => Effect.scoped( @@ -460,16 +460,14 @@ it.live("terminates a detached child when readiness is interrupted", () => new URL("../tests/host-process-fixture.ts", import.meta.url), ); const marker = path.join(root, "slow-handshake.pid"); - const markerReady = yield* waitForMarker(marker).pipe( - Effect.forkChild({ startImmediately: true }), - ); + const markerReady = yield* waitForMarker(marker); const launch = yield* launchHost(state, { stateRoot: root, cacheRoot: root, stackId: "stack", entrypoint, }).pipe(Effect.forkChild); - const pid = yield* Fiber.join(markerReady); + const pid = yield* markerReady; expect(Number.isInteger(pid) && pid > 0).toBe(true); expect(() => process.kill(pid, 0)).not.toThrow(); yield* Fiber.interrupt(launch); diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index ee9079d3a0..25fa86bae0 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -31,6 +31,7 @@ import { makeContainerRuntime, resolveEngineTarget } from "./runtime/Container.t import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; import { shutdownOwner, watchLeaseRelease } from "../tests/owner.ts"; import { noContainerClaims } from "../tests/claims.ts"; +import { watchEntry } from "../tests/watch-entry.ts"; const shortRegistrationPollFixture = fileURLToPath( new URL("../tests/short-registration-poll-fixture.ts", import.meta.url), @@ -596,14 +597,9 @@ it.live.skipIf(process.platform === "win32")( // Subscribes to the drain-deadline wait's own entry marker before sending the signal that // triggers it, so the registration deletion below never races the gate itself. - const waiting = yield* fs.watch(gateDir).pipe( - Stream.filter((event) => event.path === "waiting"), - Stream.take(1), - Stream.runDrain, - Effect.forkChild({ startImmediately: true }), - ); + const waiting = yield* watchEntry(gateDir, "waiting", true); process.kill(access.endpoint.pid, "SIGTERM"); - yield* Fiber.join(waiting).pipe(Effect.timeout("30 seconds")); + yield* waiting.pipe(Effect.timeout("30 seconds")); yield* fs.remove(`${stateRoot}/${stackId}/state.json`); yield* fs.writeFileString(`${gateDir}/release`, ""); @@ -878,14 +874,9 @@ it.live.skipIf(process.platform === "win32")( const inFlight = yield* openPartialMailSend(port, "drain-docker-inflight"); - const waiting = yield* fs.watch(gateDir).pipe( - Stream.filter((event) => event.path === "waiting"), - Stream.take(1), - Stream.runDrain, - Effect.forkChild({ startImmediately: true }), - ); + const waiting = yield* watchEntry(gateDir, "waiting", true); process.kill(access.endpoint.pid, "SIGTERM"); - yield* Fiber.join(waiting).pipe(Effect.timeout("30 seconds")); + yield* waiting.pipe(Effect.timeout("30 seconds")); expect(yield* isHttpRefused(port), "a new connection is refused once draining begins").toBe( true, @@ -992,14 +983,9 @@ it.live.skipIf(process.platform === "win32")( (connection) => Effect.sync(() => connection.destroy()), ); - const waiting = yield* fs.watch(gateDir).pipe( - Stream.filter((event) => event.path === "waiting"), - Stream.take(1), - Stream.runDrain, - Effect.forkChild({ startImmediately: true }), - ); + const waiting = yield* watchEntry(gateDir, "waiting", true); process.kill(access.endpoint.pid, "SIGTERM"); - yield* Fiber.join(waiting).pipe(Effect.timeout("30 seconds")); + yield* waiting.pipe(Effect.timeout("30 seconds")); yield* sql.unsafe("SELECT 1"); diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts index 2dd2bb91a2..00f233be00 100644 --- a/packages/stack/src/StackHost.integration.test.ts +++ b/packages/stack/src/StackHost.integration.test.ts @@ -40,6 +40,7 @@ import { shutdownOwner } from "../tests/owner.ts"; import { postgres } from "./Commands.ts"; import * as CommandRunner from "./host/CommandRunner.ts"; import { noContainerClaims } from "../tests/claims.ts"; +import { watchEntry } from "../tests/watch-entry.ts"; class HostTestError extends Data.TaggedError("HostTestError")<{ readonly message: string }> {} @@ -1325,14 +1326,9 @@ it.live( // Subscribes to the drain-deadline wait's own entry marker before sending the signal, so // the refusal check below never races the gate itself. - const waiting = yield* fs.watch(gateDir).pipe( - Stream.filter((event) => event.path === "waiting"), - Stream.take(1), - Stream.runDrain, - Effect.forkChild({ startImmediately: true }), - ); + const waiting = yield* watchEntry(gateDir, "waiting", true); process.kill(access.endpoint.pid, "SIGTERM"); - yield* Fiber.join(waiting).pipe(Effect.timeout("30 seconds")); + yield* waiting.pipe(Effect.timeout("30 seconds")); expect(yield* isHttpRefused(port), "a new connection is refused once draining begins").toBe( true, diff --git a/packages/stack/src/lifetime.integration.test.ts b/packages/stack/src/lifetime.integration.test.ts index 6643b12ec3..d68a26d701 100644 --- a/packages/stack/src/lifetime.integration.test.ts +++ b/packages/stack/src/lifetime.integration.test.ts @@ -1,17 +1,6 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { - Context, - Data, - Effect, - Fiber, - FileSystem, - Layer, - Option, - Path, - Schema, - Stream, -} from "effect"; +import { Context, Data, Effect, Fiber, FileSystem, Layer, Option, Schema, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; @@ -24,6 +13,7 @@ import { shutdownOwner, watchLeaseRelease, } from "../tests/owner.ts"; +import { watchEntry } from "../tests/watch-entry.ts"; class LifetimeTestError extends Data.TaggedError("LifetimeTestError")<{ readonly message: string; @@ -87,20 +77,6 @@ const alive = (pid: number) => { } }; -/** Completes once `entry` disappears from `directory`; subscribe before triggering the removal. */ -const awaitRemoval = (directory: string, entry: string) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const target = path.join(directory, entry); - yield* fs.watch(directory).pipe( - Stream.filter((event) => event.path === entry || event.path === target), - Stream.mapEffect(() => fs.exists(target)), - Stream.takeUntil((exists) => !exists), - Stream.runDrain, - ); - }); - it.live.skipIf(process.platform === "win32")( "destroys a session stack and its native processes when the creating process is killed", () => @@ -153,12 +129,10 @@ it.live.skipIf(process.platform === "win32")( const owned = yield* descendantsOf(ownerPid); expect(owned.length, "the owner runs the native mail service").toBeGreaterThan(0); - const removed = yield* awaitRemoval(stateRoot, stackId).pipe( - Effect.forkChild({ startImmediately: true }), - ); + const removed = yield* watchEntry(stateRoot, stackId, false); const released = yield* watchLeaseRelease(stateRoot, stackId); yield* creator.kill({ killSignal: "SIGKILL" }); - yield* Fiber.join(removed).pipe( + yield* removed.pipe( Effect.timeoutOrElse({ duration: "1 minute", orElse: () => diff --git a/packages/stack/tests/gated-drain-fixture.ts b/packages/stack/tests/gated-drain-fixture.ts index 4943fd13a9..7229971d5d 100644 --- a/packages/stack/tests/gated-drain-fixture.ts +++ b/packages/stack/tests/gated-drain-fixture.ts @@ -1,8 +1,9 @@ -import { Cause, Effect, Exit, Fiber, FileSystem, Option, Path, Schema, Stream } from "effect"; +import { Cause, Effect, Exit, FileSystem, Option, Schema } from "effect"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- readiness is an inherited launcher descriptor. import { closeSync, writeSync } from "node:fs"; import * as Network from "../src/Network.ts"; import { runStackHost, StackHostError } from "../src/StackHost.ts"; +import { watchEntry } from "./watch-entry.ts"; /** * A real owner entrypoint, identical to `internal/host-process.ts`, except it overrides @@ -14,21 +15,13 @@ import { runStackHost, StackHostError } from "../src/StackHost.ts"; const awaitRelease = (gateDir: string) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const target = path.join(gateDir, "release"); // Subscribes before signaling "waiting" below, so a `release` written the instant a test // observes that signal can never land in the gap between the signal and this watch attaching. - const released = yield* fs.watch(gateDir).pipe( - Stream.filter((event) => event.path === "release" || event.path === target), - Stream.take(1), - Stream.runDrain, - Effect.forkChild({ startImmediately: true }), - ); + const released = yield* watchEntry(gateDir, "release", true); // Marks entry into the drain-deadline wait itself, so a test can subscribe to this instead // of a sleep before it is safe to act on the assumption that drain has actually begun. - yield* fs.writeFileString(path.join(gateDir, "waiting"), ""); - if (yield* fs.exists(target)) return; - yield* Fiber.join(released); + yield* fs.writeFileString(`${gateDir}/waiting`, ""); + yield* released; }); const writeLine = (value: unknown) => diff --git a/packages/stack/tests/watch-entry.ts b/packages/stack/tests/watch-entry.ts new file mode 100644 index 0000000000..731a08f122 --- /dev/null +++ b/packages/stack/tests/watch-entry.ts @@ -0,0 +1,38 @@ +import { Effect, Path } from "effect"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem.watch's async stat before attaching makes attachment unobservable. +import { existsSync, watch } from "node:fs"; + +/** + * Watches `entry` under `directory` and returns, within the caller's scope, an effect that + * completes once the entry's existence equals `present`. Subscribe before triggering the change, + * then await the returned effect after the trigger: attaching the watcher happens synchronously + * here, and existence is only checked afterward, so no event between attachment and the trigger + * can be missed. This does not use `FileSystem.watch`: its `stat` runs asynchronously before the + * underlying `fs.watch` attaches, so attachment itself is unobservable and an event in that gap + * is lost until the caller's timeout fires. + */ +export const watchEntry = Effect.fn("Test.watchEntry")(function* ( + directory: string, + entry: string, + present: boolean, +) { + const path = yield* Path.Path; + const target = path.join(directory, entry); + const listeners = new Set<() => void>(); + yield* Effect.acquireRelease( + Effect.sync(() => watch(directory, () => listeners.forEach((listener) => listener()))), + (watcher) => Effect.sync(() => watcher.close()), + ); + return yield* Effect.succeed( + Effect.callback((resume) => { + const check = () => { + if (existsSync(target) === present) resume(Effect.void); + }; + check(); + listeners.add(check); + return Effect.sync(() => { + listeners.delete(check); + }); + }), + ); +}); From cf304732cb0fbc6ae4cafabd908e989cd067ae6b Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 15:47:06 +0200 Subject: [PATCH 041/142] refactor(stack): track containers by labels only and let a failed destroy stay registered --- .../stack/destroy/SIDE_EFFECTS.md | 7 +- .../stack/src/Commands.integration.test.ts | 5 +- packages/stack/src/Owner.ts | 94 ++-- ...ost.container-shutdown.integration.test.ts | 2 - ....container-sweep-retry.integration.test.ts | 477 ------------------ ....directory-claim-sweep.integration.test.ts | 234 +++++++++ .../stack/src/StackHost.integration.test.ts | 76 +-- packages/stack/src/StackHost.ts | 60 +-- packages/stack/src/Sweep.integration.test.ts | 6 - .../create.owner-rollback.integration.test.ts | 40 +- ...oy.runtime-unavailable.integration.test.ts | 76 +-- ...dRunner.initialization.integration.test.ts | 2 - ...dRunner.native-cleanup.integration.test.ts | 2 - packages/stack/src/host/CommandRunner.ts | 6 - packages/stack/src/namespace/Claims.ts | 36 +- .../src/runtime/Container.integration.test.ts | 62 +-- packages/stack/src/runtime/Container.ts | 37 +- ...container-image-mirror.integration.test.ts | 13 - ...nalyticsVectorImgproxy.integration.test.ts | 7 +- .../services/AuthStorage.integration.test.ts | 7 +- .../src/services/Catalog.integration.test.ts | 7 +- packages/stack/src/services/Catalog.ts | 2 - .../src/services/Database.integration.test.ts | 12 +- packages/stack/src/services/Database.ts | 4 - .../DatabaseSnapshot.integration.test.ts | 3 - .../services/Functions.integration.test.ts | 13 +- .../src/services/Mail.integration.test.ts | 7 +- .../src/services/Pooler.integration.test.ts | 7 +- .../ProcessRecipe.integration.test.ts | 13 +- .../src/services/Realtime.integration.test.ts | 7 +- packages/stack/src/services/Recipe.ts | 4 +- .../src/services/Rest.integration.test.ts | 7 +- .../src/services/Studio.integration.test.ts | 7 +- .../src/services/Vector.integration.test.ts | 7 +- .../DockerDatabaseStorage.integration.test.ts | 114 ----- .../src/storage/DockerDatabaseStorage.ts | 29 +- packages/stack/tests/claims.ts | 10 +- .../tests/docker-fixture.integration.test.ts | 3 +- .../docker-storage-publish-loop-fixture.ts | 3 - 39 files changed, 372 insertions(+), 1136 deletions(-) delete mode 100644 packages/stack/src/StackHost.container-sweep-retry.integration.test.ts create mode 100644 packages/stack/src/StackHost.directory-claim-sweep.integration.test.ts diff --git a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md index 82a226fd8b..d48d837d7d 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md @@ -32,9 +32,10 @@ reached, destruction removes nothing: the registration, port claims and host data stay as they are, stderr warns that Docker resources were not removed, and the command exits 0 reporting the stack as not destroyed. Run destroy again once Docker is reachable. Any other engine failure, an owner starting during -destruction, or resource claims the owner cannot reconcile (for example a -container on a different daemon) fails the command and keeps the stack -registered; the error lists the remaining claims and the `claims.json` path. +destruction, a directory claim the owner cannot reconcile (for example one +outside every owned root), or a labelled container that survives removal, +fails the command and keeps the stack registered; the error lists what +remains, including the `claims.json` path for any directory claims. ## Files and network diff --git a/packages/stack/src/Commands.integration.test.ts b/packages/stack/src/Commands.integration.test.ts index bef7989f20..e4ac5f7da2 100644 --- a/packages/stack/src/Commands.integration.test.ts +++ b/packages/stack/src/Commands.integration.test.ts @@ -25,7 +25,7 @@ import { makeDatabase } from "./services/Database.ts"; import { makeStandaloneService } from "../tests/standalone-service.ts"; import { bindTcp, serveTcp } from "./Proxy.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; -import { noContainerClaims, noDirectoryClaims } from "../tests/claims.ts"; +import { noDirectoryClaims } from "../tests/claims.ts"; import { dockerEngineTarget } from "../tests/engine-target.ts"; const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; @@ -43,7 +43,6 @@ const makeTestCommandRunner = (options: { Layer.build( CommandRunner.layer({ ...options, - claims: noContainerClaims, ...(options.runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), }), ).pipe(Effect.map((context) => Context.get(context, CommandRunner.Service))); @@ -90,7 +89,6 @@ describe("finite PostgreSQL commands", { timeout: 180_000 }, () => { ? yield* makeDockerDatabaseRoot("stack-tools-", stackId) : yield* fs.makeTempDirectoryScoped({ prefix: "stack-tools-" }); const database = yield* makeDatabase({ - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, root, cacheRoot, @@ -243,7 +241,6 @@ describe("finite PostgreSQL commands", { timeout: 180_000 }, () => { ? yield* makeDockerDatabaseRoot(`stack-pgprove-${major}-`, stackId) : yield* fs.makeTempDirectoryScoped({ prefix: `stack-pgprove-${major}-` }); const database = yield* makeDatabase({ - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, root, cacheRoot, diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index eaeccdcc1c..665c401c20 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -103,10 +103,13 @@ const directoryClaimExists = Effect.fn("Owner.directoryClaimExists")(function* ( }); /** - * Removes this stack's claimed resources by exact identity, through the owner's pinned engine target. - * Container claims from another or unknown daemon, and directory claims outside every owned root, - * are kept and reported, so recovery (possibly as root) never deletes what it does not own. A claim - * naming a path that no longer exists in any form is simply dropped, nothing left to remove. + * Reconciles this stack's directory claims by exact identity, then removes every container + * carrying this stack's identity and data-root labels through the owner's pinned engine target. + * A directory claim outside every owned root is kept and reported, so recovery (possibly as root) + * never deletes what it does not own; one naming a path that no longer exists in any form is + * simply dropped, nothing left to remove. A container that survives removal fails the sweep, so + * the caller (startup, stop, or destroy) can report it and retry instead of proceeding as if the + * stack were fully torn down. */ export const sweepContainers = Effect.fn("Owner.sweepContainers")(function* ( state: StackNamespace.Interface, @@ -123,41 +126,38 @@ export const sweepContainers = Effect.fn("Owner.sweepContainers")(function* ( yield* Claims.reconcile( state, saved.id, - ( - claim, - ): Effect.Effect< - "removed" | "kept", - Container.ContainerError | PlatformError | StackNamespace.NamespaceError, - ChildProcessSpawner.ChildProcessSpawner - > => { - if (claim.kind === "directory") - return directoryClaimExists(claim.id).pipe( - Effect.flatMap((present) => - !present - ? Effect.succeed("removed" as const) - : Paths.isWithinOwnedRoots(fs, path, claim.id, ownedRoots, (_operation, cause) => - namespaceError("cleanup", cause), - ).pipe( - Effect.flatMap((within) => - within - ? fs - .remove(claim.id, { recursive: true, force: true }) - .pipe(Effect.as("removed" as const)) - : Effect.logWarning( - `Claimed directory ${claim.id} resolves outside every owned root; keeping it`, - ).pipe(Effect.as("kept" as const)), - ), + (claim): Effect.Effect<"removed" | "kept", PlatformError | StackNamespace.NamespaceError> => + directoryClaimExists(claim.id).pipe( + Effect.flatMap((present) => + !present + ? Effect.succeed("removed" as const) + : Paths.isWithinOwnedRoots(fs, path, claim.id, ownedRoots, (_operation, cause) => + namespaceError("cleanup", cause), + ).pipe( + Effect.flatMap((within) => + within + ? fs + .remove(claim.id, { recursive: true, force: true }) + .pipe(Effect.as("removed" as const)) + : Effect.logWarning( + `Claimed directory ${claim.id} resolves outside every owned root; keeping it`, + ).pipe(Effect.as("kept" as const)), ), - ), - ); - if (target === undefined) return Effect.succeed("kept" as const); - if (claim.daemonId === undefined || claim.daemonId !== target.daemonId) - return Effect.succeed("kept" as const); - return Container.removeContainerById({ target, id: claim.id }).pipe( - Effect.as("removed" as const), - ); - }, + ), + ), + ), ); + if (target === undefined) return; + const remaining = yield* Container.removeStackContainers({ + target, + stackId: saved.id, + stackRoot: path.resolve(root), + }); + if (remaining.length > 0) + return yield* new StackNamespace.NamespaceError({ + operation: "cleanup", + message: `Containers remain: ${remaining.join(", ")}`, + }); }); type NamespaceError = @@ -491,7 +491,6 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { cacheRoot: options.cacheRoot, runtime, helpers, - containerClaims: claims.containers, directoryClaims: claims.directories, isPubliclyReserved, ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), @@ -892,9 +891,9 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { Effect.andThen(network.drain), Effect.andThen(orchestrator.destroyNamespace), Effect.andThen(sweep), - // A claim reconcile deliberately kept (for example one recorded against a different - // daemon) must not be lost to a full deregistration; the stack stays registered so a - // later acquisition's reconcile, against the right daemon, can still finish it. + // A directory claim deliberately kept (for example one resolving outside every owned + // root) must not be lost to a full deregistration; the stack stays registered so a later + // acquisition's reconcile can still finish it. Effect.andThen(options.state.readClaims(stackId)), Effect.flatMap((remaining) => remaining.length === 0 @@ -903,22 +902,15 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { Effect.andThen(options.state.remove(stackId)), ) : Effect.gen(function* () { - const currentDaemonId = options.engineTarget?.daemonId; const claimsPath = path.join(options.state.root, stackId, Claims.CLAIMS_FILE); - const listed = remaining - .map( - (claim) => - ` - ${claim.kind} ${claim.id} (daemon ${claim.daemonId ?? "unknown"})`, - ) - .join("\n"); + const listed = remaining.map((claim) => ` - ${claim.kind} ${claim.id}`).join("\n"); return yield* new StackNamespace.NamespaceError({ operation: "destroy", message: [ `${remaining.length} resource claim(s) could not be reconciled; stack stays registered for retry.`, listed, - `Current daemon: ${currentDaemonId ?? "unreachable"}.`, - "To recover: stop this owner, confirm the lease is free, then run `supabase stack destroy` again once the original engine is back. " + - `If that engine is permanently gone, remove the listed claim entries from ${claimsPath} and retry.`, + `To recover: confirm the listed directories are safe to remove, then run \`supabase stack destroy\` again. ` + + `If they are not, remove the listed claim entries from ${claimsPath} and retry.`, ].join("\n"), }); }), diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index 25fa86bae0..9d6e868f6e 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -30,7 +30,6 @@ import { import { makeContainerRuntime, resolveEngineTarget } from "./runtime/Container.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; import { shutdownOwner, watchLeaseRelease } from "../tests/owner.ts"; -import { noContainerClaims } from "../tests/claims.ts"; import { watchEntry } from "../tests/watch-entry.ts"; const shortRegistrationPollFixture = fileURLToPath( @@ -221,7 +220,6 @@ it.live.skipIf(process.platform === "win32")( const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const target = yield* resolveEngineTarget(spawner); const helper = yield* makeContainerRuntime({ - claims: noContainerClaims, target, root: dataA, }); diff --git a/packages/stack/src/StackHost.container-sweep-retry.integration.test.ts b/packages/stack/src/StackHost.container-sweep-retry.integration.test.ts deleted file mode 100644 index 1a52a7bf73..0000000000 --- a/packages/stack/src/StackHost.container-sweep-retry.integration.test.ts +++ /dev/null @@ -1,477 +0,0 @@ -import { NodeHttpClient, NodeServices } from "@effect/platform-node"; -import { expect, it } from "@effect/vitest"; -import { Context, Effect, FileSystem, Layer, Path, Sink, Stream } from "effect"; -import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import * as StackNamespace from "./StackNamespace.ts"; -import * as Owner from "./Owner.ts"; -import type { EngineTarget } from "./runtime/Container.ts"; -import { NativeRuntimeRootBase, nativeRuntimeRootPath } from "./runtime/postgres-user.ts"; - -const stateFor = (root: string) => - Layer.build(StackNamespace.layer({ root })).pipe( - Effect.map((context) => Context.get(context, StackNamespace.Service)), - ); - -const containerId = "claimed0000000000000000000000000001"; - -/** Resolved once, like an owner's own startup would, and pinned for the whole test. */ -const engineTarget: EngineTarget = { engine: "docker", argv: [], daemonId: "current-daemon" }; - -it.live( - "retains a claim when the engine is unreachable during destroy, and finishes it once the engine recovers", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-container-sweep-retry-" }); - const state = yield* stateFor(`${root}/state`); - const saved = { - id: "stack", - runtime: "docker" as const, - identity: { projectRoot: root, branchContext: "main", stackName: "sweep-retry" }, - instances: [], - lifetime: "detached" as const, - composition: { members: [], dependencies: [] }, - ports: [], - }; - yield* state.save(saved); - // A now-dead owner already claimed this container before being interrupted. Recorded - // against the daemon this reconcile run will later match, once it recovers. - yield* state.claim(saved.id, { - kind: "container", - id: containerId, - daemonId: "current-daemon", - }); - let engineReachable = false; - let rmCalls = 0; - const engine = ChildProcessSpawner.make((command) => { - if (!ChildProcess.isStandardCommand(command) || command.command !== "docker") - return Effect.die("Unexpected child process command"); - const isClaimRemoval = command.args[0] === "rm" && command.args.includes(containerId); - if (isClaimRemoval) rmCalls++; - const fail = isClaimRemoval && !engineReachable; - return Effect.succeed( - ChildProcessSpawner.makeHandle({ - pid: ChildProcessSpawner.ProcessId(0), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(fail ? 1 : 0)), - isRunning: Effect.succeed(false), - kill: () => Effect.void, - stdin: Sink.drain, - stdout: Stream.empty, - stderr: fail - ? Stream.succeed( - new TextEncoder().encode( - "Cannot connect to the Docker daemon at tcp://127.0.0.1:1. Is the docker daemon running?", - ), - ) - : Stream.empty, - all: Stream.empty, - getInputFd: () => Sink.drain, - getOutputFd: () => Stream.empty, - unref: Effect.succeed(Effect.void), - }), - ); - }); - const ownerLayer = Owner.layer({ - saved, - root: `${root}/data`, - cacheRoot: `${root}/cache`, - engineTarget, - }).pipe( - Layer.provide( - Layer.merge( - Layer.succeed(StackNamespace.Service, state), - Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, engine), - ), - ), - ); - const firstOwner = yield* Layer.build(ownerLayer).pipe( - Effect.map((context) => Context.get(context, Owner.Service)), - ); - - const firstAttempt = yield* firstOwner.namespace.destroy.pipe(Effect.exit); - expect(firstAttempt._tag).toBe("Failure"); - expect(rmCalls).toBe(1); - expect(yield* state.read(saved.id)).toBeDefined(); - expect((yield* state.readClaims(saved.id)).map((claim) => claim.id)).toEqual([containerId]); - - // The engine is reachable again: a later acquisition's reconcile finishes the cleanup. - engineReachable = true; - const retryOwner = yield* Layer.build(ownerLayer).pipe( - Effect.map((context) => Context.get(context, Owner.Service)), - ); - yield* retryOwner.namespace.destroy; - expect(rmCalls).toBe(2); - expect(yield* state.read(saved.id)).toBeUndefined(); - expect(yield* state.readClaims(saved.id)).toEqual([]); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), -); - -it.live( - "removes a claimed directory left behind under /tmp and unclaims it, for a native stack with no engine", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-directory-sweep-" }); - const state = yield* stateFor(`${root}/state`); - const saved = { - id: "stack", - runtime: "native" as const, - identity: { projectRoot: root, branchContext: "main", stackName: "directory-sweep" }, - instances: [], - lifetime: "detached" as const, - composition: { members: [], dependencies: [] }, - ports: [], - }; - yield* state.save(saved); - // A now-dead owner claimed this short-socket directory, nested under the native runtime - // root, before being killed; not scoped, since the reconcile under test is itself the one - // that removes it. - const runtimeRootBase = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-directory-sweep-runtime-root-", - }); - const runtimeRoot = nativeRuntimeRootPath(path, runtimeRootBase, process.getuid?.() ?? 0); - const socketDirectory = path.join(runtimeRoot, "pg-leftover"); - yield* fs.makeDirectory(socketDirectory, { recursive: true, mode: 0o700 }); - yield* fs.writeFileString(`${socketDirectory}/pg_hba.conf`, "local all all trust\n"); - yield* state.claim(saved.id, { kind: "directory", id: socketDirectory }); - const ownerLayer = Owner.layer({ - saved, - root: `${root}/data`, - cacheRoot: `${root}/cache`, - }).pipe( - Layer.provide( - Layer.merge( - Layer.succeed(StackNamespace.Service, state), - Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), - ), - ), - ), - Layer.provide(Layer.succeed(NativeRuntimeRootBase, runtimeRootBase)), - ); - const owner = yield* Layer.build(ownerLayer).pipe( - Effect.map((context) => Context.get(context, Owner.Service)), - ); - yield* owner.namespace.destroy; - expect(yield* fs.exists(socketDirectory)).toBe(false); - expect(yield* state.readClaims(saved.id)).toEqual([]); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), -); - -it.live( - "keeps and reports a claimed directory that resolves outside every owned root, for a native stack with no engine", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-directory-sweep-outside-", - }); - const state = yield* stateFor(`${root}/state`); - const saved = { - id: "stack", - runtime: "native" as const, - identity: { - projectRoot: root, - branchContext: "main", - stackName: "directory-sweep-outside", - }, - instances: [], - lifetime: "detached" as const, - composition: { members: [], dependencies: [] }, - ports: [], - }; - yield* state.save(saved); - // A foreign directory under neither the data root nor the native runtime root: never ours - // by location, however it got its name, so recovery must leave it alone. - const foreign = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-directory-sweep-foreign-", - }); - yield* fs.writeFileString(`${foreign}/marker`, "not ours\n"); - yield* state.claim(saved.id, { kind: "directory", id: foreign }); - const ownerLayer = Owner.layer({ - saved, - root: `${root}/data`, - cacheRoot: `${root}/cache`, - }).pipe( - Layer.provide( - Layer.merge( - Layer.succeed(StackNamespace.Service, state), - Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), - ), - ), - ), - ); - const owner = yield* Layer.build(ownerLayer).pipe( - Effect.map((context) => Context.get(context, Owner.Service)), - ); - const failure = yield* owner.namespace.destroy.pipe(Effect.flip); - expect(failure.message).toContain(`directory ${foreign} (daemon unknown)`); - expect(yield* fs.exists(foreign)).toBe(true); - expect(yield* state.read(saved.id)).toBeDefined(); - expect((yield* state.readClaims(saved.id)).map((claim) => claim.id)).toEqual([foreign]); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), -); - -it.live( - "keeps and reports a claim nested under a runtime root a foreign uid could have taken over", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-directory-sweep-takeover-", - }); - const state = yield* stateFor(`${root}/state`); - const saved = { - id: "stack", - runtime: "native" as const, - identity: { - projectRoot: root, - branchContext: "main", - stackName: "directory-sweep-takeover", - }, - instances: [], - lifetime: "detached" as const, - composition: { members: [], dependencies: [] }, - ports: [], - }; - yield* state.save(saved); - // Tests can't create another uid's files, so a group/world-writable leaf stands in for one - // recreated by a foreign uid after the original was removed. - const runtimeRootBase = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-directory-sweep-takeover-base-", - }); - const runtimeRoot = nativeRuntimeRootPath(path, runtimeRootBase, process.getuid?.() ?? 0); - const claimed = path.join(runtimeRoot, "pg-claimed"); - yield* fs.makeDirectory(claimed, { recursive: true }); - yield* fs.chmod(runtimeRoot, 0o777); - yield* fs.writeFileString(`${claimed}/marker`, "not ours\n"); - yield* state.claim(saved.id, { kind: "directory", id: claimed }); - const ownerLayer = Owner.layer({ - saved, - root: `${root}/data`, - cacheRoot: `${root}/cache`, - }).pipe( - Layer.provide( - Layer.merge( - Layer.succeed(StackNamespace.Service, state), - Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), - ), - ), - ), - Layer.provide(Layer.succeed(NativeRuntimeRootBase, runtimeRootBase)), - ); - const owner = yield* Layer.build(ownerLayer).pipe( - Effect.map((context) => Context.get(context, Owner.Service)), - ); - const failure = yield* owner.namespace.destroy.pipe(Effect.flip); - expect(failure.message).toContain(`directory ${claimed} (daemon unknown)`); - expect(yield* fs.exists(claimed)).toBe(true); - expect((yield* state.readClaims(saved.id)).map((claim) => claim.id)).toEqual([claimed]); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), -); - -it.live("drops a claimed directory that no longer exists, for a native stack with no engine", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-directory-sweep-gone-" }); - const state = yield* stateFor(`${root}/state`); - const saved = { - id: "stack", - runtime: "native" as const, - identity: { projectRoot: root, branchContext: "main", stackName: "directory-sweep-gone" }, - instances: [], - lifetime: "detached" as const, - composition: { members: [], dependencies: [] }, - ports: [], - }; - yield* state.save(saved); - // Never created on disk: the owner that journaled it crashed before the directory existed. - const gone = path.join(root, "pg-never-existed"); - yield* state.claim(saved.id, { kind: "directory", id: gone }); - const ownerLayer = Owner.layer({ - saved, - root: `${root}/data`, - cacheRoot: `${root}/cache`, - }).pipe( - Layer.provide( - Layer.merge( - Layer.succeed(StackNamespace.Service, state), - Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), - ), - ), - ), - ); - const owner = yield* Layer.build(ownerLayer).pipe( - Effect.map((context) => Context.get(context, Owner.Service)), - ); - yield* owner.namespace.destroy; - expect(yield* state.readClaims(saved.id)).toEqual([]); - expect(yield* state.read(saved.id)).toBeUndefined(); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), -); - -it.live( - "keeps a container claim recorded against a different daemon, never removing or dropping it", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-container-sweep-daemon-", - }); - const state = yield* stateFor(`${root}/state`); - const saved = { - id: "stack", - runtime: "docker" as const, - identity: { projectRoot: root, branchContext: "main", stackName: "sweep-daemon" }, - instances: [], - lifetime: "detached" as const, - composition: { members: [], dependencies: [] }, - ports: [], - }; - yield* state.save(saved); - // Recorded against a daemon this reconcile run will never match. - yield* state.claim(saved.id, { - kind: "container", - id: containerId, - daemonId: "other-daemon", - }); - let rmCalls = 0; - const engine = ChildProcessSpawner.make((command) => { - if (!ChildProcess.isStandardCommand(command) || command.command !== "docker") - return Effect.die("Unexpected child process command"); - if (command.args[0] === "rm" && command.args.includes(containerId)) rmCalls++; - return Effect.succeed( - ChildProcessSpawner.makeHandle({ - pid: ChildProcessSpawner.ProcessId(0), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), - isRunning: Effect.succeed(false), - kill: () => Effect.void, - stdin: Sink.drain, - stdout: Stream.empty, - stderr: Stream.empty, - all: Stream.empty, - getInputFd: () => Sink.drain, - getOutputFd: () => Stream.empty, - unref: Effect.succeed(Effect.void), - }), - ); - }); - const ownerLayer = Owner.layer({ - saved, - root: `${root}/data`, - cacheRoot: `${root}/cache`, - engineTarget, - }).pipe( - Layer.provide( - Layer.merge( - Layer.succeed(StackNamespace.Service, state), - Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, engine), - ), - ), - ); - const owner = yield* Layer.build(ownerLayer).pipe( - Effect.map((context) => Context.get(context, Owner.Service)), - ); - const failure = yield* owner.namespace.destroy.pipe(Effect.flip); - expect(rmCalls).toBe(0); - expect(failure.message).toContain(`container ${containerId} (daemon other-daemon)`); - expect(failure.message).toContain("Current daemon: current-daemon."); - expect(failure.message).toContain(`${root}/state/stack/claims.json`); - // Kept, not dropped: a full deregistration would otherwise have discarded it unreconciled. - expect(yield* state.read(saved.id)).toBeDefined(); - expect((yield* state.readClaims(saved.id)).map((claim) => claim.id)).toEqual([containerId]); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), -); - -it.live( - "keeps a container claim with no recorded daemon id, even once the current daemon is known", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-container-sweep-no-daemon-", - }); - const state = yield* stateFor(`${root}/state`); - const saved = { - id: "stack", - runtime: "docker" as const, - identity: { projectRoot: root, branchContext: "main", stackName: "sweep-no-daemon" }, - instances: [], - lifetime: "detached" as const, - composition: { members: [], dependencies: [] }, - ports: [], - }; - yield* state.save(saved); - // No daemon id recorded at all, unlike a mismatched one: nothing to compare the current, - // successfully-resolved daemon against, so this is kept too, not removed by assuming it's - // the same daemon. - yield* state.claim(saved.id, { kind: "container", id: containerId }); - let rmCalls = 0; - const engine = ChildProcessSpawner.make((command) => { - if (!ChildProcess.isStandardCommand(command) || command.command !== "docker") - return Effect.die("Unexpected child process command"); - if (command.args[0] === "rm" && command.args.includes(containerId)) rmCalls++; - return Effect.succeed( - ChildProcessSpawner.makeHandle({ - pid: ChildProcessSpawner.ProcessId(0), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), - isRunning: Effect.succeed(false), - kill: () => Effect.void, - stdin: Sink.drain, - stdout: Stream.empty, - stderr: Stream.empty, - all: Stream.empty, - getInputFd: () => Sink.drain, - getOutputFd: () => Stream.empty, - unref: Effect.succeed(Effect.void), - }), - ); - }); - const ownerLayer = Owner.layer({ - saved, - root: `${root}/data`, - cacheRoot: `${root}/cache`, - engineTarget, - }).pipe( - Layer.provide( - Layer.merge( - Layer.succeed(StackNamespace.Service, state), - Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, engine), - ), - ), - ); - const owner = yield* Layer.build(ownerLayer).pipe( - Effect.map((context) => Context.get(context, Owner.Service)), - ); - const failure = yield* owner.namespace.destroy.pipe(Effect.flip); - expect(rmCalls).toBe(0); - expect(failure.message).toContain(`container ${containerId} (daemon unknown)`); - expect(failure.message).toContain("Current daemon: current-daemon."); - expect(yield* state.read(saved.id)).toBeDefined(); - expect((yield* state.readClaims(saved.id)).map((claim) => claim.id)).toEqual([containerId]); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), -); diff --git a/packages/stack/src/StackHost.directory-claim-sweep.integration.test.ts b/packages/stack/src/StackHost.directory-claim-sweep.integration.test.ts new file mode 100644 index 0000000000..c9d69f3752 --- /dev/null +++ b/packages/stack/src/StackHost.directory-claim-sweep.integration.test.ts @@ -0,0 +1,234 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { Context, Effect, FileSystem, Layer, Path } from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import * as StackNamespace from "./StackNamespace.ts"; +import * as Owner from "./Owner.ts"; +import { NativeRuntimeRootBase, nativeRuntimeRootPath } from "./runtime/postgres-user.ts"; + +const stateFor = (root: string) => + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), + ); + +it.live( + "removes a claimed directory left behind under /tmp and unclaims it, for a native stack with no engine", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-directory-sweep-" }); + const state = yield* stateFor(`${root}/state`); + const saved = { + id: "stack", + runtime: "native" as const, + identity: { projectRoot: root, branchContext: "main", stackName: "directory-sweep" }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + // A now-dead owner claimed this short-socket directory, nested under the native runtime + // root, before being killed; not scoped, since the reconcile under test is itself the one + // that removes it. + const runtimeRootBase = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-directory-sweep-runtime-root-", + }); + const runtimeRoot = nativeRuntimeRootPath(path, runtimeRootBase, process.getuid?.() ?? 0); + const socketDirectory = path.join(runtimeRoot, "pg-leftover"); + yield* fs.makeDirectory(socketDirectory, { recursive: true, mode: 0o700 }); + yield* fs.writeFileString(`${socketDirectory}/pg_hba.conf`, "local all all trust\n"); + yield* state.claim(saved.id, { kind: "directory", id: socketDirectory }); + const ownerLayer = Owner.layer({ + saved, + root: `${root}/data`, + cacheRoot: `${root}/cache`, + }).pipe( + Layer.provide( + Layer.merge( + Layer.succeed(StackNamespace.Service, state), + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), + ), + ), + ), + Layer.provide(Layer.succeed(NativeRuntimeRootBase, runtimeRootBase)), + ); + const owner = yield* Layer.build(ownerLayer).pipe( + Effect.map((context) => Context.get(context, Owner.Service)), + ); + yield* owner.namespace.destroy; + expect(yield* fs.exists(socketDirectory)).toBe(false); + expect(yield* state.readClaims(saved.id)).toEqual([]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live( + "keeps and reports a claimed directory that resolves outside every owned root, for a native stack with no engine", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-directory-sweep-outside-", + }); + const state = yield* stateFor(`${root}/state`); + const saved = { + id: "stack", + runtime: "native" as const, + identity: { + projectRoot: root, + branchContext: "main", + stackName: "directory-sweep-outside", + }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + // A foreign directory under neither the data root nor the native runtime root: never ours + // by location, however it got its name, so recovery must leave it alone. + const foreign = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-directory-sweep-foreign-", + }); + yield* fs.writeFileString(`${foreign}/marker`, "not ours\n"); + yield* state.claim(saved.id, { kind: "directory", id: foreign }); + const ownerLayer = Owner.layer({ + saved, + root: `${root}/data`, + cacheRoot: `${root}/cache`, + }).pipe( + Layer.provide( + Layer.merge( + Layer.succeed(StackNamespace.Service, state), + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), + ), + ), + ), + ); + const owner = yield* Layer.build(ownerLayer).pipe( + Effect.map((context) => Context.get(context, Owner.Service)), + ); + const failure = yield* owner.namespace.destroy.pipe(Effect.flip); + expect(failure.message).toContain(`directory ${foreign}`); + expect(yield* fs.exists(foreign)).toBe(true); + expect(yield* state.read(saved.id)).toBeDefined(); + expect((yield* state.readClaims(saved.id)).map((claim) => claim.id)).toEqual([foreign]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live( + "keeps and reports a claim nested under a runtime root a foreign uid could have taken over", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-directory-sweep-takeover-", + }); + const state = yield* stateFor(`${root}/state`); + const saved = { + id: "stack", + runtime: "native" as const, + identity: { + projectRoot: root, + branchContext: "main", + stackName: "directory-sweep-takeover", + }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + // Tests can't create another uid's files, so a group/world-writable leaf stands in for one + // recreated by a foreign uid after the original was removed. + const runtimeRootBase = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-directory-sweep-takeover-base-", + }); + const runtimeRoot = nativeRuntimeRootPath(path, runtimeRootBase, process.getuid?.() ?? 0); + const claimed = path.join(runtimeRoot, "pg-claimed"); + yield* fs.makeDirectory(claimed, { recursive: true }); + yield* fs.chmod(runtimeRoot, 0o777); + yield* fs.writeFileString(`${claimed}/marker`, "not ours\n"); + yield* state.claim(saved.id, { kind: "directory", id: claimed }); + const ownerLayer = Owner.layer({ + saved, + root: `${root}/data`, + cacheRoot: `${root}/cache`, + }).pipe( + Layer.provide( + Layer.merge( + Layer.succeed(StackNamespace.Service, state), + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), + ), + ), + ), + Layer.provide(Layer.succeed(NativeRuntimeRootBase, runtimeRootBase)), + ); + const owner = yield* Layer.build(ownerLayer).pipe( + Effect.map((context) => Context.get(context, Owner.Service)), + ); + const failure = yield* owner.namespace.destroy.pipe(Effect.flip); + expect(failure.message).toContain(`directory ${claimed}`); + expect(yield* fs.exists(claimed)).toBe(true); + expect((yield* state.readClaims(saved.id)).map((claim) => claim.id)).toEqual([claimed]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live("drops a claimed directory that no longer exists, for a native stack with no engine", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-directory-sweep-gone-" }); + const state = yield* stateFor(`${root}/state`); + const saved = { + id: "stack", + runtime: "native" as const, + identity: { projectRoot: root, branchContext: "main", stackName: "directory-sweep-gone" }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + // Never created on disk: the owner that journaled it crashed before the directory existed. + const gone = path.join(root, "pg-never-existed"); + yield* state.claim(saved.id, { kind: "directory", id: gone }); + const ownerLayer = Owner.layer({ + saved, + root: `${root}/data`, + cacheRoot: `${root}/cache`, + }).pipe( + Layer.provide( + Layer.merge( + Layer.succeed(StackNamespace.Service, state), + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), + ), + ), + ), + ); + const owner = yield* Layer.build(ownerLayer).pipe( + Effect.map((context) => Context.get(context, Owner.Service)), + ); + yield* owner.namespace.destroy; + expect(yield* state.readClaims(saved.id)).toEqual([]); + expect(yield* state.read(saved.id)).toBeUndefined(); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts index 00f233be00..ee5b6b3a05 100644 --- a/packages/stack/src/StackHost.integration.test.ts +++ b/packages/stack/src/StackHost.integration.test.ts @@ -39,7 +39,6 @@ import { bindControl, makeRuntime } from "./StackHost.ts"; import { shutdownOwner } from "../tests/owner.ts"; import { postgres } from "./Commands.ts"; import * as CommandRunner from "./host/CommandRunner.ts"; -import { noContainerClaims } from "../tests/claims.ts"; import { watchEntry } from "../tests/watch-entry.ts"; class HostTestError extends Data.TaggedError("HostTestError")<{ readonly message: string }> {} @@ -236,7 +235,6 @@ const inProcessRuntime = ( const acquired = yield* bindControl(); const toolContext = yield* Layer.build( CommandRunner.layer({ - claims: noContainerClaims, stackId: "stack", root, cacheRoot: "/tmp/supabase-stack-artifacts", @@ -527,7 +525,7 @@ it.live( ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); -it.live("reports destroy and fallback stop failures together", () => +it.live("reports a destroy failure and leaves the owner running", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -549,68 +547,42 @@ it.live("reports destroy and fallback stop failures together", () => root: `${root}/data`, cacheRoot: "/tmp/supabase-stack-artifacts", }); - const failureWithOutcome = ( - operation: "destroy" | "stop", - message: string, - id: string, - reason: string, - ) => - new OrchestratorError({ - operation, - message, - outcomes: [ - { - id, - result: Exit.fail(new OrchestratorError({ operation, message: reason })), - }, - ], - }); const failedOwner = { ...owner, namespace: { ...owner.namespace, destroy: Effect.fail( - failureWithOutcome( - "destroy", - "Namespace destroy had failures", - "database-destroy", - "data removal refused", - ), - ), - stop: Effect.fail( - failureWithOutcome( - "stop", - "Composition stop had failures", - "database-stop", - "process stop refused", - ), + new OrchestratorError({ + operation: "destroy", + message: "Namespace destroy had failures", + outcomes: [ + { + id: "database-destroy", + result: Exit.fail( + new OrchestratorError({ + operation: "destroy", + message: "data removal refused", + }), + ), + }, + ], + }), ), }, }; const { runtime } = yield* inProcessRuntime(failedOwner, state, root); const failure = yield* shutdownOwner(runtime.access, true).pipe(Effect.flip); expect(failure.message).toContain("Namespace destroy had failures"); - expect(failure.message).toContain("database-destroy:"); - expect(failure.message).toContain("data removal refused"); - expect(failure.message).toContain("fallback stop failed: Composition stop had failures"); - expect(failure.message).toContain("database-stop:"); - expect(failure.message).toContain("process stop refused"); expect("outcomes" in failure).toBe(true); if (!("outcomes" in failure)) return yield* Effect.die("shutdown outcomes were missing"); - expect(failure.outcomes).toEqual( - expect.arrayContaining([ - { - id: "database-destroy", - succeeded: false, - error: expect.stringContaining("data removal refused"), - }, - { - id: "database-stop", - succeeded: false, - error: expect.stringContaining("process stop refused"), - }, - ]), - ); + expect(failure.outcomes).toEqual([ + { + id: "database-destroy", + succeeded: false, + error: expect.stringContaining("data removal refused"), + }, + ]); + // A failed destroy does not fall back to stop: retry destroy or call `stack stop` instead. expect(yield* Deferred.isDone(runtime.exit)).toBe(false); expect(yield* owner.getServing).toBe(true); }), diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index 88da4f221e..34354edcde 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -42,7 +42,6 @@ import { } from "./HostProcess.ts"; import { ChildProcessSpawner } from "effect/unstable/process"; import { projectSegmentFor } from "./identity/Identity.ts"; -import * as Claims from "./namespace/Claims.ts"; import * as Owner from "./Owner.ts"; import { StackError, stackError, StackRpc, type RunCommandPayload } from "./Rpc.ts"; import { makeHostGateway, resolveEngineTarget } from "./runtime/Container.ts"; @@ -288,7 +287,6 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( yield* runner.cleanup; yield* owner.namespace.stop; }); - let retiringAfterDestroyFailure = false; const body: Effect.Effect = mode === "abandon" ? Effect.gen(function* () { @@ -318,57 +316,16 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( Effect.mapError((cause) => stackError("shutdown", cause)), ) : Effect.gen(function* () { - const destroyExit = yield* Effect.gen(function* () { - yield* attachments.stopAll; - yield* runner.cleanup; - yield* owner.namespace.destroy; - }).pipe(Effect.exit); - if (Exit.isSuccess(destroyExit)) { - yield* finish; - return; - } - const stopExit = yield* stopOwned.pipe(Effect.exit); - if (Exit.isFailure(stopExit)) { - const describeCause = (cause: Cause.Cause) => { - const error = Option.match(Cause.findErrorOption(cause), { - onNone: () => - new StackError({ - operation: "shutdown", - message: Cause.pretty(cause), - }), - onSome: (value) => stackError("shutdown", value), - }); - const failedOutcomes = error.outcomes - ?.filter(({ succeeded }) => !succeeded) - .map(({ id, error: reason }) => `${id}: ${reason ?? "failed"}`) - .join("; "); - return { - error, - message: - failedOutcomes === undefined || failedOutcomes.length === 0 - ? error.message - : `${error.message} (${failedOutcomes})`, - }; - }; - const destroyFailure = describeCause(destroyExit.cause); - const stopFailure = describeCause(stopExit.cause); - const outcomes = [ - ...(destroyFailure.error.outcomes ?? []), - ...(stopFailure.error.outcomes ?? []), - ]; - return yield* new StackError({ - operation: "shutdown", - message: `${destroyFailure.message}; fallback stop failed: ${stopFailure.message}`, - ...(outcomes.length === 0 ? {} : { outcomes }), - }); - } - retiringAfterDestroyFailure = true; - yield* finish; - return yield* Effect.failCause(destroyExit.cause); - }).pipe(Effect.mapError((cause) => stackError("shutdown", cause))); + yield* attachments.stopAll; + yield* runner.cleanup; + yield* owner.namespace.destroy; + }).pipe( + Effect.andThen(finish), + Effect.mapError((cause) => stackError("shutdown", cause)), + ); const cleanup = body.pipe( Effect.catchCause((cause) => - mode !== "abandon" && !retiringAfterDestroyFailure + mode !== "abandon" ? owner .setDraining(false) // Drain may already have run (and so left the network's own admission @@ -648,7 +605,6 @@ export const runStackHost = Effect.fn("StackHost.run")( root: dataRoot, cacheRoot: options.cacheRoot, runtime: saved.runtime, - claims: Claims.forStack(state, saved.id).containers, hostGateway, engineTarget, }), diff --git a/packages/stack/src/Sweep.integration.test.ts b/packages/stack/src/Sweep.integration.test.ts index bc874b89ec..2dbadb7efa 100644 --- a/packages/stack/src/Sweep.integration.test.ts +++ b/packages/stack/src/Sweep.integration.test.ts @@ -22,7 +22,6 @@ import { makeContainerRuntime, resolveEngineTarget } from "./runtime/Container.t import * as StackNamespace from "./StackNamespace.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; import { shutdownOwner, watchLeaseRelease } from "../tests/owner.ts"; -import { noContainerClaims } from "../tests/claims.ts"; class SweepTestError extends Data.TaggedError("SweepTestError")<{ readonly message: string }> {} @@ -174,7 +173,6 @@ it.live.skipIf(process.platform === "win32")( const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const target = yield* resolveEngineTarget(spawner); const helper = yield* makeContainerRuntime({ - claims: noContainerClaims, target, root: dataA, }); @@ -188,10 +186,6 @@ it.live.skipIf(process.platform === "win32")( const dead = (yield* launchHost(state, { stateRoot: rootA, cacheRoot, stackId: deadId })) .endpoint; const orphan = yield* createOwnedContainer(deadId, dataA); - // The dead owner claimed this container before it was killed; the reconcile loop that - // replaces label-based sweeping only ever acts on recorded claims, never on labels alone. - // Recorded against this real daemon's own id: a claim with none is now always kept. - yield* state.claim(deadId, { kind: "container", id: orphan, daemonId: target.daemonId }); const otherRoot = yield* createOwnedContainer(deadId, dataB); const deadReleased = yield* watchLeaseRelease(rootA, deadId); yield* Effect.sync(() => process.kill(dead.pid, "SIGKILL")); diff --git a/packages/stack/src/create.owner-rollback.integration.test.ts b/packages/stack/src/create.owner-rollback.integration.test.ts index c038be8ad8..c309180f93 100644 --- a/packages/stack/src/create.owner-rollback.integration.test.ts +++ b/packages/stack/src/create.owner-rollback.integration.test.ts @@ -1,32 +1,12 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Context, Effect, Exit, Fiber, FileSystem, Layer, Path } from "effect"; +import { Effect, Exit, Fiber, FileSystem, Layer, Path } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { create, discover } from "./effect.ts"; import { deriveStackId, resolveStackIdentity } from "./identity/Identity.ts"; -import * as StackNamespace from "./StackNamespace.ts"; const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); -/** Seeds a container claim for a not-yet-created stack id, as an orphaned claim would be found. */ -const seedContainerClaim = ( - stateRoot: string, - id: string, - containerId: string, - daemonId?: string, -) => - Effect.scoped( - Layer.build(StackNamespace.layer({ root: stateRoot })).pipe( - Effect.flatMap((context) => - Context.get(context, StackNamespace.Service).claim(id, { - kind: "container", - id: containerId, - ...(daemonId === undefined ? {} : { daemonId }), - }), - ), - ), - ); - it.live( "removes a stack it just registered when the owner fails to launch, and lets a retry succeed", () => @@ -61,9 +41,6 @@ it.live( } satisfies Parameters[0]; const identity = yield* resolveStackIdentity(options); const id = yield* deriveStackId(identity); - // An orphaned claim from a never-fully-registered prior attempt, found by this owner's - // startup reconcile before it finishes registering. - yield* seedContainerClaim(options.stateRoot, id, "rollback0000000000000000000000001"); const launchFailure = yield* Effect.flip(create({ ...options, startOwner: true })); expect(launchFailure.message).toContain("Cannot connect to the Docker daemon"); @@ -95,11 +72,14 @@ it.live("removes a stack it just registered when its owner launch is interrupted const binDir = path.join(root, "bin"); yield* fs.makeDirectory(binDir, { recursive: true }); const dockerShim = path.join(binDir, "docker"); + // An orphaned container from a never-fully-registered prior attempt, found by this owner's + // startup label sweep before it finishes registering. + const orphanId = "interrupt0000000000000000000000001"; yield* fs.writeFileString( dockerShim, // The leading check strips the pinned `--host ` prefix every invocation now // carries (fixed below via DOCKER_HOST), so "$1" below still sees the real command. - `#!/bin/sh\nif [ "$1" = "--host" ]; then shift 2; fi\nif [ "$1" = "rm" ]; then echo started > '${started}'; exec sleep 60; fi\nif [ "$1" = "info" ]; then echo fake-daemon-id; fi\nexit 0\n`, + `#!/bin/sh\nif [ "$1" = "--host" ]; then shift 2; fi\nif [ "$1" = "ps" ]; then echo '${orphanId}'; exit 0; fi\nif [ "$1" = "rm" ]; then echo started > '${started}'; exec sleep 60; fi\nif [ "$1" = "info" ]; then echo fake-daemon-id; fi\nexit 0\n`, ); yield* fs.chmod(dockerShim, 0o755); // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the detached host subprocess inherits PATH; this is not application config. @@ -129,16 +109,6 @@ it.live("removes a stack it just registered when its owner launch is interrupted cacheRoot: `${root}/cache`, runtime: "docker", } satisfies Parameters[0]; - const identity = yield* resolveStackIdentity(options); - const id = yield* deriveStackId(identity); - // Recorded against the daemon this reconcile run will match: the engine is reachable and - // identifiable here, so the interrupted launch's own rollback is what's under test. - yield* seedContainerClaim( - options.stateRoot, - id, - "interrupt0000000000000000000000001", - "fake-daemon-id", - ); const creating = yield* create({ ...options, startOwner: true }).pipe(Effect.forkChild); yield* fs.readFileString(started); yield* Fiber.interrupt(creating); diff --git a/packages/stack/src/destroy.runtime-unavailable.integration.test.ts b/packages/stack/src/destroy.runtime-unavailable.integration.test.ts index b32a78ea29..1611c4f512 100644 --- a/packages/stack/src/destroy.runtime-unavailable.integration.test.ts +++ b/packages/stack/src/destroy.runtime-unavailable.integration.test.ts @@ -1,8 +1,7 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Context, Effect, FileSystem, Layer, Path } from "effect"; +import { Effect, FileSystem, Layer, Path } from "effect"; import { create, discover } from "./effect.ts"; -import * as StackNamespace from "./StackNamespace.ts"; const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); @@ -43,39 +42,8 @@ const shimDocker = Effect.fn("shimDocker")(function* (root: string, script: stri ); }); -/** Opens the real namespace at `stateRoot`, bypassing any owner, to seed or read claims directly. */ -const withNamespace = ( - stateRoot: string, - use: (state: StackNamespace.Interface) => Effect.Effect, -) => - Effect.scoped( - Layer.build(StackNamespace.layer({ root: stateRoot })).pipe( - Effect.flatMap((context) => use(Context.get(context, StackNamespace.Service))), - ), - ); - -/** Records a container claim as if a now-dead owner had created it before being interrupted. */ -const seedContainerClaim = ( - stateRoot: string, - id: string, - containerId: string, - daemonId?: string, -) => - withNamespace(stateRoot, (state) => - state.claim(id, { - kind: "container", - id: containerId, - ...(daemonId === undefined ? {} : { daemonId }), - }), - ); - -const readClaimIds = (stateRoot: string, id: string) => - withNamespace(stateRoot, (state) => state.readClaims(id)).pipe( - Effect.map((claims) => claims.map((claim) => claim.id)), - ); - it.live( - "keeps a stack's registration and claims when destroy finds no owner and its engine is unreachable, then finishes the cleanup once a later owner can start", + "keeps a stack registered when destroy finds no owner and its engine is unreachable, then finishes the cleanup once a later owner can start", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -92,23 +60,12 @@ it.live( runtime: "docker", } satisfies Parameters[0]; const stack = yield* create(options); - // Recorded against the daemon the engine will report once it is back: the second destroy - // attempt below, once the engine is reachable again, is expected to finish the cleanup. - yield* seedContainerClaim( - options.stateRoot, - stack.id, - "offline0000000000000000000000000001", - "fake-daemon-id", - ); const result = yield* stack.destroy; expect(result).toEqual({ runtimeCleanup: "skipped", engine: "docker" }); expect(yield* discover({ stateRoot: options.stateRoot })).toHaveLength(1); - expect(yield* readClaimIds(options.stateRoot, stack.id)).toEqual([ - "offline0000000000000000000000000001", - ]); - // The engine is back: destroy retried finishes the exact claimed cleanup automatically. + // The engine is back: destroy retried finishes the cleanup automatically. yield* shimDocker( root, '#!/bin/sh\nif [ "$1" = "info" ]; then\n echo \'fake-daemon-id\'\nfi\nexit 0\n', @@ -135,7 +92,6 @@ it.live("keeps a stack registered when Windows reports its Docker daemon pipe is cacheRoot: `${root}/cache`, runtime: "docker", }); - yield* seedContainerClaim(stateRoot, stack.id, "windows000000000000000000000000001"); const result = yield* stack.destroy; @@ -155,7 +111,6 @@ it.live("keeps a stack registered when its engine CLI is not installed", () => cacheRoot: `${root}/cache`, runtime: "docker", }); - yield* seedContainerClaim(stateRoot, stack.id, "nocli00000000000000000000000000001"); yield* fs.makeDirectory(`${root}/empty-bin`); // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the detached host subprocess inherits PATH; this is not application config. const originalPath = process.env.PATH; @@ -190,7 +145,6 @@ it.live("keeps a stack registered when Docker reports its API socket is missing" cacheRoot: `${root}/cache`, runtime: "docker", }); - yield* seedContainerClaim(stateRoot, stack.id, "socket0000000000000000000000000001"); const result = yield* stack.destroy; @@ -215,7 +169,6 @@ it.live("keeps a stack registered when its container engine rejects the listing" runtime: "docker", } satisfies Parameters[0]; const stack = yield* create(options); - yield* seedContainerClaim(options.stateRoot, stack.id, "permission00000000000000000000001"); const destroyFailure = yield* Effect.flip(stack.destroy); // The socket itself refuses the owner's own startup, resolving its pinned engine target: @@ -223,9 +176,6 @@ it.live("keeps a stack registered when its container engine rejects the listing" expect(destroyFailure.message).toContain("permission denied"); expect(yield* discover({ stateRoot: options.stateRoot })).toHaveLength(1); - expect(yield* readClaimIds(options.stateRoot, stack.id)).toEqual([ - "permission00000000000000000000001", - ]); }).pipe(Effect.scoped, Effect.provide(layer)), ); @@ -233,15 +183,21 @@ it.live("keeps a stack registered when its container engine is reachable but cle Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-cleanup-failure-" }); + // The daemon is reachable and identifiable, and its label listing finds this stack's own + // container; only the targeted removal fails. + const containerId = "cleanupfail0000000000000000000001"; yield* shimDocker( root, [ "#!/bin/sh", + 'if [ "$1" = "ps" ]; then', + ` echo '${containerId}'`, + " exit 0", + "fi", 'if [ "$1" = "rm" ]; then', " echo 'docker: Error response from daemon: container removal failed' >&2", " exit 1", "fi", - // The daemon is reachable and identifiable; only the targeted removal fails. 'if [ "$1" = "info" ]; then', " echo 'fake-daemon-id'", "fi", @@ -257,22 +213,10 @@ it.live("keeps a stack registered when its container engine is reachable but cle runtime: "docker", } satisfies Parameters[0]; const stack = yield* create(options); - // Recorded against the daemon this reconcile run will match: the engine is reachable and - // identifiable here, so only the targeted removal itself is expected to fail. - yield* seedContainerClaim( - options.stateRoot, - stack.id, - "cleanupfail0000000000000000000001", - "fake-daemon-id", - ); const destroyFailure = yield* Effect.flip(stack.destroy); expect(destroyFailure.message).toContain("container removal failed"); expect(yield* discover({ stateRoot: options.stateRoot })).toHaveLength(1); - // Cleanup authority never transferred away from the claim, so a later retry can still finish it. - expect(yield* readClaimIds(options.stateRoot, stack.id)).toEqual([ - "cleanupfail0000000000000000000001", - ]); }).pipe(Effect.scoped, Effect.provide(layer)), ); diff --git a/packages/stack/src/host/CommandRunner.initialization.integration.test.ts b/packages/stack/src/host/CommandRunner.initialization.integration.test.ts index 8aa2bccadb..840ca45fcd 100644 --- a/packages/stack/src/host/CommandRunner.initialization.integration.test.ts +++ b/packages/stack/src/host/CommandRunner.initialization.integration.test.ts @@ -12,7 +12,6 @@ import { import { PreparationError } from "../preparation/Errors.ts"; import type { StackCredentials } from "../StackNamespace.ts"; import * as CommandRunner from "./CommandRunner.ts"; -import { noContainerClaims } from "../../tests/claims.ts"; const target = process.platform === "darwin" && process.arch === "arm64" @@ -67,7 +66,6 @@ const prepareAuthArtifact = Effect.fn(function* (cacheRoot: string, script: stri const makeRunner = Effect.fn(function* (root: string, cacheRoot: string) { const layer = CommandRunner.layer({ - claims: noContainerClaims, stackId: "initialization-command-test", root, cacheRoot, diff --git a/packages/stack/src/host/CommandRunner.native-cleanup.integration.test.ts b/packages/stack/src/host/CommandRunner.native-cleanup.integration.test.ts index 448abeea55..e806b20aec 100644 --- a/packages/stack/src/host/CommandRunner.native-cleanup.integration.test.ts +++ b/packages/stack/src/host/CommandRunner.native-cleanup.integration.test.ts @@ -12,7 +12,6 @@ import { } from "../preparation/ArtifactStore.ts"; import { PreparationError } from "../preparation/Errors.ts"; import * as CommandRunner from "./CommandRunner.ts"; -import { noContainerClaims } from "../../tests/claims.ts"; const target = process.platform === "darwin" && process.arch === "arm64" @@ -114,7 +113,6 @@ it.live.skipIf(target === undefined || process.platform === "win32")( }), ); const layer = CommandRunner.layer({ - claims: noContainerClaims, stackId: "native-cleanup-test", root, cacheRoot, diff --git a/packages/stack/src/host/CommandRunner.ts b/packages/stack/src/host/CommandRunner.ts index 5308b7d5be..a0eabf3faf 100644 --- a/packages/stack/src/host/CommandRunner.ts +++ b/packages/stack/src/host/CommandRunner.ts @@ -21,7 +21,6 @@ import { resolveArtifact, useNativeArtifact, } from "../Artifacts.ts"; -import type * as Claims from "../namespace/Claims.ts"; import * as Environment from "../namespace/Environment.ts"; import { borrow } from "../namespace/Paths.ts"; import { makeContainerRuntime, type EngineTarget, type HostGateway } from "../runtime/Container.ts"; @@ -82,8 +81,6 @@ const makeCommandRunner = (options: { readonly hostGateway?: HostGateway; /** The engine endpoint and identity the owner resolved once at startup; absent when native. */ readonly engineTarget?: EngineTarget; - /** Journals each job container before it is created, for the namespace's reconcile loop. */ - readonly claims: Claims.ContainerClaims; }) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -100,7 +97,6 @@ const makeCommandRunner = (options: { root: options.root, imageMirrors: slimImageMirrors, ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), - claims: options.claims, }); const borrowCallerPath = (candidate: string) => borrow(fs, path, candidate, options.root, (_operation, cause) => failure(cause)); @@ -338,6 +334,4 @@ export const layer = (options: { readonly hostGateway?: HostGateway; /** The engine endpoint and identity the owner resolved once at startup; absent when native. */ readonly engineTarget?: EngineTarget; - /** Journals each job container before it is created, for the namespace's reconcile loop. */ - readonly claims: Claims.ContainerClaims; }) => Layer.effect(Service, makeCommandRunner(options).pipe(Effect.map(Service.of))); diff --git a/packages/stack/src/namespace/Claims.ts b/packages/stack/src/namespace/Claims.ts index 494bcc0170..30e2daf8b7 100644 --- a/packages/stack/src/namespace/Claims.ts +++ b/packages/stack/src/namespace/Claims.ts @@ -3,15 +3,12 @@ import { namespaceError, type NamespaceError } from "./Capabilities.ts"; import * as Publication from "./Publication.ts"; /** - * A resource the stack is about to create outside its own process lifetime: a container or a - * directory whose creation can be interrupted partway. Identity is exact, so reconciliation never - * guesses by name or label. `daemonId` records which engine daemon a container claim belongs to, - * so reconcile never removes (or mistakes) an identity recorded against a different daemon. + * A directory the stack is about to create outside its own process lifetime, whose creation can + * be interrupted partway. Identity is exact, so reconciliation never guesses by name or label. */ export const ResourceClaim = Schema.Struct({ - kind: Schema.Literals(["container", "directory"]), + kind: Schema.Literals(["directory"]), id: Schema.String, - daemonId: Schema.optionalKey(Schema.String), }); export interface ResourceClaim extends Schema.Schema.Type {} @@ -125,33 +122,17 @@ export const make = ( return { claim, unclaim, readClaims }; }); -/** A narrow capability a container runtime uses to journal exact container identities. */ -export interface ContainerClaims { - /** `daemonId`, when known, is recorded so reconcile never drops a claim seen from another daemon. */ - readonly claim: (id: string, daemonId?: string) => Effect.Effect; - readonly unclaim: (id: string) => Effect.Effect; -} - /** A narrow capability for journaling a directory outside the owner's normal data tree. */ export interface DirectoryClaims { readonly claim: (directoryPath: string) => Effect.Effect; readonly unclaim: (directoryPath: string) => Effect.Effect; } -/** Scopes the claims journal to one stack, exposing only its container and directory operations. */ +/** Scopes the claims journal to one stack, exposing only its directory operations. */ export const forStack = ( state: Interface, stackId: string, -): { readonly containers: ContainerClaims; readonly directories: DirectoryClaims } => ({ - containers: { - claim: (id, daemonId) => - state.claim(stackId, { - kind: "container", - id, - ...(daemonId === undefined ? {} : { daemonId }), - }), - unclaim: (id) => state.unclaim(stackId, { kind: "container", id }), - }, +): { readonly directories: DirectoryClaims } => ({ directories: { claim: (directoryPath) => state.claim(stackId, { kind: "directory", id: directoryPath }), unclaim: (directoryPath) => state.unclaim(stackId, { kind: "directory", id: directoryPath }), @@ -160,10 +141,9 @@ export const forStack = ( /** * Reads a stack's recorded claims and, for each one still present, asks `remove` to reconcile it: - * `"removed"` drops the claim, `"kept"` leaves it (for example a container claim recorded against - * a different daemon than the one reconcile is currently running against). A claim whose removal - * fails stays recorded, along with every claim after it, for the next acquisition to retry; it - * reports the first failure. + * `"removed"` drops the claim, `"kept"` leaves it. A claim whose removal fails stays recorded, + * along with every claim after it, for the next acquisition to retry; it reports the first + * failure. */ export const reconcile = Effect.fn("Namespace.Claims.reconcile")(function* ( claimsInterface: Pick, diff --git a/packages/stack/src/runtime/Container.integration.test.ts b/packages/stack/src/runtime/Container.integration.test.ts index 4d390566ba..d7c242cc90 100644 --- a/packages/stack/src/runtime/Container.integration.test.ts +++ b/packages/stack/src/runtime/Container.integration.test.ts @@ -39,7 +39,6 @@ import * as Owner from "../Owner.ts"; import * as StackNamespace from "../StackNamespace.ts"; import { makeDockerDatabaseStorage } from "../storage/DockerDatabaseStorage.ts"; import { reconcileContainerPassword } from "../services/Database.ts"; -import { noContainerClaims } from "../../tests/claims.ts"; const image = await Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -65,7 +64,6 @@ describe("container process adapter", () => { Effect.gen(function* () { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -77,7 +75,6 @@ describe("container process adapter", () => { const pullAttempted = yield* Ref.make(false); const spawner = makePullFailureSpawner(delegate, pullAttempted); yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }).pipe( @@ -96,7 +93,6 @@ describe("container process adapter", () => { const pullAttempted = yield* Ref.make(false); const spawner = makePullFailureSpawner(delegate, pullAttempted); const result = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }).pipe( @@ -116,7 +112,6 @@ describe("container process adapter", () => { const id = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -154,7 +149,6 @@ describe("container process adapter", () => { const id = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -194,7 +188,6 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", hostGateway: yield* makeHostGateway, @@ -233,7 +226,6 @@ describe("container process adapter", () => { const probes = yield* Ref.make(0); const hostGateway = yield* makeHostGateway; const makeRuntime = makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", hostGateway, @@ -279,14 +271,12 @@ describe("container process adapter", () => { const hostGateway = yield* makeHostGateway; const spawner = makeHostGatewayProbeSpawner(delegate, probes, () => undefined, release); const database = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", hostGateway, awaitHostGateway: false, }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)); const service = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", hostGateway, @@ -322,7 +312,6 @@ describe("container process adapter", () => { Effect.gen(function* () { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", hostGateway: yield* makeHostGateway, @@ -359,7 +348,6 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", hostGateway: yield* makeHostGateway, @@ -398,7 +386,6 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", hostGateway: yield* makeHostGateway, @@ -438,7 +425,6 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", hostGateway: yield* makeHostGateway, @@ -471,7 +457,6 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", hostGateway: yield* makeHostGateway, @@ -507,7 +492,6 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", hostGateway: yield* makeHostGateway, @@ -547,7 +531,6 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const database = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", hostGateway: yield* makeHostGateway, @@ -581,7 +564,6 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -609,7 +591,6 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -636,7 +617,6 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -670,7 +650,6 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -710,7 +689,6 @@ describe("container process adapter", () => { it.live("stops a container with its configured stop signal", () => Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -740,7 +718,6 @@ describe("container process adapter", () => { it.live("waits until a discarded container stops before returning", () => Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -771,7 +748,6 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -803,7 +779,6 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -835,7 +810,6 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const released = yield* Deferred.make(); const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }).pipe( @@ -877,7 +851,6 @@ describe("container process adapter", () => { yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }).pipe( @@ -911,7 +884,6 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -947,7 +919,6 @@ describe("container process adapter", () => { const result = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -988,7 +959,6 @@ describe("container process adapter", () => { const result = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -1034,7 +1004,6 @@ describe("container process adapter", () => { const result = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -1084,7 +1053,6 @@ describe("container process adapter", () => { const result = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -1128,7 +1096,6 @@ describe("container process adapter", () => { const result = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -1179,7 +1146,6 @@ describe("container process adapter", () => { const result = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -1227,7 +1193,6 @@ describe("container process adapter", () => { const result = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -1284,7 +1249,6 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -1329,7 +1293,6 @@ describe("container process adapter", () => { const result = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -1373,7 +1336,6 @@ describe("container process adapter", () => { const result = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -1406,7 +1368,6 @@ describe("container process adapter", () => { Effect.gen(function* () { const engine = yield* makeHangingCreateSpawner(); const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }).pipe(Effect.provide(engine.layer)); @@ -1432,7 +1393,6 @@ describe("container process adapter", () => { Effect.gen(function* () { const engine = yield* makeHangingCreateSpawner(); const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }).pipe(Effect.provide(engine.layer)); @@ -1591,6 +1551,9 @@ describe("container process adapter", () => { const root = yield* fs.makeTempDirectoryScoped({ prefix: "engine-target-pin-" }); const commands: Array> = []; let activeContext = "alpha"; + // An orphaned container found by label alone: listed once (for removal), then gone. + const orphanId = "orphan00000000000000000000000000001"; + let psCalls = 0; const handle = (exitCode: number, stdout = "") => ChildProcessSpawner.makeHandle({ pid: ChildProcessSpawner.ProcessId(0), @@ -1620,6 +1583,13 @@ describe("container process adapter", () => { if (unpinned[0] === "inspect") return Effect.succeed(handle(0, '{"Ports":{},"Status":"running","ExitCode":0}')); if (unpinned[0] === "run") return Effect.succeed(handle(0, "abcdef012345")); + // Two call sites share "ps": the launched container's own `--rm` removal poll (filtered + // by name, always confirmed gone) and the label-based orphan discovery below. + if (unpinned[0] === "ps" && unpinned.some((arg) => arg.includes("com.supabase.stack="))) { + psCalls += 1; + return Effect.succeed(handle(0, psCalls === 1 ? `${orphanId}\n` : "")); + } + if (unpinned[0] === "ps") return Effect.succeed(handle(0, "")); return Effect.succeed(handle(0)); }); @@ -1633,7 +1603,6 @@ describe("container process adapter", () => { commands.length = 0; const container = yield* makeContainerRuntime({ - claims: noContainerClaims, target, root, }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)); @@ -1654,7 +1623,6 @@ describe("container process adapter", () => { // Helper creation, sharing the same container runtime a real stack would. const storage = yield* makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target, stackId: "engine-target-pin", @@ -1678,8 +1646,8 @@ describe("container process adapter", () => { spawner, ); - // Reconcile: an orphaned claim from a crashed owner, removed through the same target. - const orphanId = "orphan00000000000000000000000000001"; + // Reconcile: an orphaned container, found and removed by label alone, through the same + // pinned target. const state = Context.get( yield* Layer.build(StackNamespace.layer({ root: path.join(root, "state") })), StackNamespace.Service, @@ -1694,15 +1662,9 @@ describe("container process adapter", () => { ports: [], }; yield* state.save(saved); - yield* state.claim(saved.id, { - kind: "container", - id: orphanId, - daemonId: target.daemonId, - }); yield* Owner.sweepContainers(state, saved, root, target).pipe( Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), ); - expect(yield* state.readClaims(saved.id)).toEqual([]); // Every call these consumers made, after the target was resolved, carries the context // pinned at startup: asserted of the whole list, not merely of a filtered subset, so a diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index 1a98876d9b..eef6c8b680 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -21,7 +21,6 @@ import { } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { testRunLabelArgs as readTestRunLabelArgs } from "../internal/test-run-label.ts"; -import type * as Claims from "../namespace/Claims.ts"; import { CONTAINER_ENV_DIRNAME } from "../namespace/Paths.ts"; import { identifyContainer } from "./ContainerName.ts"; @@ -360,11 +359,6 @@ export const makeContainerRuntime = (options: { * for containers that do not rely on reaching the host over IPv4. */ readonly awaitHostGateway?: boolean; - /** - * Journals each container's exact identity before it is created, so a crashed owner leaves it - * recoverable; required so a container creation that forgets to wire claims does not compile. - */ - readonly claims: Claims.ContainerClaims; }): Effect.Effect< ContainerRuntime, never, @@ -533,13 +527,8 @@ export const makeContainerRuntime = (options: { const token = yield* crypto.randomUUIDv4.pipe( Effect.mapError((cause) => errorFor("identity", cause)), ); - // Not an instance container: no instance label, so it is never mistaken for one. It is - // still named and claimed before the engine creates it, so a crashed owner's reconcile - // loop can still find and remove it instead of leaking it. + // Not an instance container: no instance label, so it is never mistaken for one. const { name } = identifyContainer(spec, token, true); - yield* options.claims - .claim(name, options.target.daemonId) - .pipe(Effect.mapError((cause) => errorFor("claim", cause))); const hosts = yield* run( [ "run", @@ -562,9 +551,6 @@ export const makeContainerRuntime = (options: { { timeout: undefined }, ); yield* awaitRemoved(name); - yield* options.claims - .unclaim(name) - .pipe(Effect.mapError((cause) => errorFor("claim", cause))); return hosts; }).pipe(Effect.timeout(HOST_GATEWAY_PROBE_TIMEOUT)); /** Resolves the IPv4 host address the engine writes itself, since it rejects `host-gateway`. */ @@ -662,13 +648,6 @@ export const makeContainerRuntime = (options: { Effect.mapError((cause) => errorFor("identity", cause)), ); const { name, composeProject, composeService } = identifyContainer(spec, token, oneOff); - // Journaled before the engine creates anything, so a crashed owner leaves an exact, - // recoverable identity instead of one the next owner must guess by label. The daemon id - // lets a later reconcile against a different daemon keep this claim rather than remove (or - // mistake) an identity that daemon has never heard of. - yield* options.claims - .claim(name, options.target.daemonId) - .pipe(Effect.mapError((cause) => errorFor("claim", cause))); const hostAlias = yield* hostAliasTarget(image, spec); const testRunLabel = yield* testRunLabelArgs; const createArgs = (target: string | undefined) => [ @@ -853,11 +832,6 @@ export const makeContainerRuntime = (options: { yield* run(["rm", name]).pipe(Effect.catchTag("ContainerError", reconcileAbsent)); yield* Ref.set(stopped, true); yield* Ref.set(removed, true); - // Cleanup authority now belongs entirely to this normal teardown; a crash before this - // point still leaves the claim for the next owner's reconcile loop. - yield* options.claims - .unclaim(name) - .pipe(Effect.mapError((cause) => errorFor("claim", cause))); }); yield* Scope.addFinalizer( owner, @@ -1023,12 +997,11 @@ export const makeContainerRuntime = (options: { }); /** - * Removes a container by its exact recorded identity, succeeding when it is already absent. The - * namespace's reconcile loop uses this instead of discovering candidates by label, through the - * same pinned {@link EngineTarget} it resolved identity through, so a context switch in between - * can never split the two. + * Removes a container by its exact id, succeeding when it is already absent, through the same + * pinned {@link EngineTarget} it was discovered through, so a context switch in between can never + * split the two. */ -export const removeContainerById = Effect.fn("Container.removeContainerById")( +const removeContainerById = Effect.fn("Container.removeContainerById")( (options: { readonly target: EngineTarget; readonly id: string }) => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; diff --git a/packages/stack/src/runtime/container-image-mirror.integration.test.ts b/packages/stack/src/runtime/container-image-mirror.integration.test.ts index a739c40662..760145b947 100644 --- a/packages/stack/src/runtime/container-image-mirror.integration.test.ts +++ b/packages/stack/src/runtime/container-image-mirror.integration.test.ts @@ -4,7 +4,6 @@ import { Cause, Effect, Exit, Fiber, Layer, Option, Sink, Stream } from "effect" import { TestClock } from "effect/testing"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { makeContainerRuntime, type EngineTarget } from "./Container.ts"; -import { noContainerClaims } from "../../tests/claims.ts"; // An unpinned target: these tests exercise image mirroring, not endpoint pinning. const dockerTarget: EngineTarget = { engine: "docker", argv: [], daemonId: "test-daemon-id" }; @@ -85,7 +84,6 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [mirror] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror] : []), @@ -104,7 +102,6 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [secondMirror] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror, secondMirror] : []), @@ -124,7 +121,6 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [mirror] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror] : []), @@ -142,7 +138,6 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [mirror] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror] : []), @@ -160,7 +155,6 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror] : []), @@ -179,7 +173,6 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [primary, mirror] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror] : []), @@ -194,7 +187,6 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [primary], throttled: { [primary]: 2 } }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -215,7 +207,6 @@ describe("container image mirror", () => { }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -241,7 +232,6 @@ describe("container image mirror", () => { }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -259,7 +249,6 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [primary], throttled: { [primary]: 10 } }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -278,7 +267,6 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [primary], throttled: { [primary]: 1 } }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", }); @@ -295,7 +283,6 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [mirror], throttled: { [primary]: 10 } }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror] : []), diff --git a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts index 5686a98f1a..cae39c79f0 100644 --- a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts +++ b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts @@ -6,11 +6,7 @@ import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { - noContainerClaims, - noDirectoryClaims, - noPublicPortReservations, -} from "../../tests/claims.ts"; +import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ @@ -19,7 +15,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }); diff --git a/packages/stack/src/services/AuthStorage.integration.test.ts b/packages/stack/src/services/AuthStorage.integration.test.ts index 981739094c..dc6118829a 100644 --- a/packages/stack/src/services/AuthStorage.integration.test.ts +++ b/packages/stack/src/services/AuthStorage.integration.test.ts @@ -7,11 +7,7 @@ import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { - noContainerClaims, - noDirectoryClaims, - noPublicPortReservations, -} from "../../tests/claims.ts"; +import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ @@ -20,7 +16,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }); diff --git a/packages/stack/src/services/Catalog.integration.test.ts b/packages/stack/src/services/Catalog.integration.test.ts index 8683d6eea2..bdf9bf0ce1 100644 --- a/packages/stack/src/services/Catalog.integration.test.ts +++ b/packages/stack/src/services/Catalog.integration.test.ts @@ -2,11 +2,7 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Effect, Exit, FileSystem, Layer, Redacted } from "effect"; import { makeServiceRecipe } from "./Catalog.ts"; -import { - noContainerClaims, - noDirectoryClaims, - noPublicPortReservations, -} from "../../tests/claims.ts"; +import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; const options = (root: string) => ({ stackId: "catalog-recipe", @@ -14,7 +10,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }); diff --git a/packages/stack/src/services/Catalog.ts b/packages/stack/src/services/Catalog.ts index d9a7eaa119..05779b8fe7 100644 --- a/packages/stack/src/services/Catalog.ts +++ b/packages/stack/src/services/Catalog.ts @@ -331,7 +331,6 @@ export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( ...(options.helpers === undefined ? {} : { helpers: options.helpers }), ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), ...(options.engineTarget === undefined ? {} : { engineTarget: options.engineTarget }), - containerClaims: options.containerClaims, directoryClaims: options.directoryClaims, }).pipe( Effect.mapError( @@ -354,7 +353,6 @@ export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( root: options.root, imageMirrors: slimImageMirrors, ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), - claims: options.containerClaims, }); const borrowCallerPath = (candidate: string) => borrow(fs, path, candidate, options.root, (operation, cause) => { diff --git a/packages/stack/src/services/Database.integration.test.ts b/packages/stack/src/services/Database.integration.test.ts index c51321da1d..047161e2a4 100644 --- a/packages/stack/src/services/Database.integration.test.ts +++ b/packages/stack/src/services/Database.integration.test.ts @@ -18,7 +18,7 @@ import { DEFAULT_POSTGRES_ROOT_KEY } from "../Defaults.ts"; import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeDatabase, type BackendEndpoint, type DatabaseConfig } from "./Database.ts"; import { makeDockerDatabaseRoot, runDocker } from "../../tests/docker-fixture.ts"; -import { noContainerClaims, noDirectoryClaims } from "../../tests/claims.ts"; +import { noDirectoryClaims } from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const config: DatabaseConfig = { @@ -72,7 +72,6 @@ describe("database component", { timeout: 180_000 }, () => { ? yield* makeDockerDatabaseRoot("stack-default-key-", "default-key-test") : yield* fs.makeTempDirectoryScoped({ prefix: "stack-default-key-" }); const recipe = yield* makeDatabase({ - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, stackId: "default-key-test", instanceId: "database", @@ -133,7 +132,6 @@ describe("database component", { timeout: 180_000 }, () => { ).toEqual([{ decrypted_secret: "preserved-value" }]); if (target.runtime === "native") { const second = yield* makeDatabase({ - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, stackId: "default-key-test", instanceId: "database-second", @@ -169,7 +167,6 @@ describe("database component", { timeout: 180_000 }, () => { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-database-hba-" }); const database = yield* makeDatabase({ - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, stackId: "stack-integration", instanceId: "hba", @@ -233,7 +230,6 @@ describe("database component", { timeout: 180_000 }, () => { const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-database-" }); const cacheRoot = artifactCacheRoot; const first = yield* makeDatabase({ - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, stackId: "stack-integration", instanceId: "first", @@ -311,7 +307,6 @@ describe("database component", { timeout: 180_000 }, () => { expect(incomplete.message).toContain("major does not match"); const reopened = yield* makeDatabase({ - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, stackId: "stack-integration", instanceId: "first", @@ -339,7 +334,6 @@ describe("database component", { timeout: 180_000 }, () => { const secondRoot = path.join(root, "second-root"); const second = yield* makeDatabase({ - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, stackId: "stack-integration", instanceId: "second", @@ -395,7 +389,6 @@ describe("database component", { timeout: 180_000 }, () => { ); const cacheRoot = artifactCacheRoot; const database = yield* makeDatabase({ - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, stackId: "stack-integration-container", instanceId: "database", @@ -453,7 +446,6 @@ describe("database component", { timeout: 180_000 }, () => { expect((yield* service.get).lifecycle).toBe("running"); yield* service.stop; const reopened = yield* makeDatabase({ - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, stackId: "stack-integration-container", instanceId: "database", @@ -491,7 +483,6 @@ describe("database component", { timeout: 180_000 }, () => { const stackId = "stack-compose-group"; const root = yield* makeDockerDatabaseRoot("stack-database-group-", stackId); const database = yield* makeDatabase({ - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, stackId, instanceId: "database", @@ -537,7 +528,6 @@ describe("database component", { timeout: 180_000 }, () => { const stackId = "stack-fast-shutdown"; const root = yield* makeDockerDatabaseRoot("stack-database-shutdown-", stackId); const database = yield* makeDatabase({ - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, stackId, instanceId: "database", diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index 7ae260d2e3..3663d29c8d 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -145,8 +145,6 @@ export interface DatabaseOptions { readonly hostGateway?: HostGateway; /** The engine endpoint and identity the owner resolved once at startup; absent when native. */ readonly engineTarget?: EngineTarget; - /** Journals this database's container before it is created, for the namespace's reconcile loop. */ - readonly containerClaims: Claims.ContainerClaims; /** Journals the native socket directory this database creates under `/tmp`, outside its data root. */ readonly directoryClaims: Claims.DirectoryClaims; } @@ -469,7 +467,6 @@ export const makeDatabase = ( root: options.root, imageMirrors: slimImageMirrors, ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), - claims: options.containerClaims, // PostgreSQL's outbound HTTP falls back across address families. awaitHostGateway: false, }); @@ -491,7 +488,6 @@ export const makeDatabase = ( container, spawner, ...(options.helpers === undefined ? {} : { helpers: options.helpers }), - claims: options.containerClaims, }).pipe(Effect.mapError((cause) => databaseError("storage", cause))); const snapshots = (version: string) => diff --git a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts index 55ac66cea7..3d3a610613 100644 --- a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts +++ b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts @@ -9,7 +9,6 @@ import { makeDockerHelperRegistry } from "../storage/DockerHelperRegistry.ts"; import { shellQuote } from "../storage/DockerSnapshotBackend.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { makeDatabaseSnapshots, type SnapshotScope } from "./DatabaseSnapshot.ts"; -import { noContainerClaims } from "../../tests/claims.ts"; // Derived from the real catalog (not hardcoded), so a Postgres pin bump never makes this go stale. const version = postgresVersion("17"); @@ -186,7 +185,6 @@ const docker = Effect.fnUntraced(function* () { const image = (yield* resolveArtifact({ service: "database", version }).pipe(Effect.orDie)).image; const target = yield* resolveEngineTarget(spawner); const container = yield* makeContainerRuntime({ - claims: noContainerClaims, target, root, }); @@ -248,7 +246,6 @@ const docker = Effect.fnUntraced(function* () { const instanceRoot = `${root}/${name}`; yield* fs.makeDirectory(instanceRoot, { recursive: true }); const storage = yield* makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target, stackId, diff --git a/packages/stack/src/services/Functions.integration.test.ts b/packages/stack/src/services/Functions.integration.test.ts index e89e07d3d9..aed0557564 100644 --- a/packages/stack/src/services/Functions.integration.test.ts +++ b/packages/stack/src/services/Functions.integration.test.ts @@ -22,11 +22,7 @@ import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import * as Functions from "./Functions.ts"; import { makeProcessRecipe } from "./ProcessRecipe.ts"; -import { - noContainerClaims, - noDirectoryClaims, - noPublicPortReservations, -} from "../../tests/claims.ts"; +import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ @@ -106,7 +102,6 @@ describe("service catalog", () => { }, }, { - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, ...dockerOptions(root), @@ -171,7 +166,6 @@ describe("service catalog", () => { config: { functionsRoot, databaseUrl: "postgres://functions-db" }, }, { - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, ...options(root), @@ -267,7 +261,6 @@ describe("service catalog", () => { }, }, { - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, ...options(stackRoot), @@ -341,7 +334,6 @@ describe("service catalog", () => { }, }, { - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, ...options(stackRoot), @@ -418,7 +410,6 @@ describe("service catalog", () => { }, }, { - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, ...options(stackRoot), @@ -534,7 +525,6 @@ for (const runtime of ["native", "docker"] as const) { }, }, { - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, ...options(stackRoot), @@ -658,7 +648,6 @@ it.effect("passes POSIX project paths to a docker Functions container from a Win const recipe = yield* makeProcessRecipe( creation, { - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, stackId: "e".repeat(64), diff --git a/packages/stack/src/services/Mail.integration.test.ts b/packages/stack/src/services/Mail.integration.test.ts index 75a456cdcd..5142dd05ab 100644 --- a/packages/stack/src/services/Mail.integration.test.ts +++ b/packages/stack/src/services/Mail.integration.test.ts @@ -4,11 +4,7 @@ import { Effect, Exit, FileSystem, Layer, Path } from "effect"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; -import { - noContainerClaims, - noDirectoryClaims, - noPublicPortReservations, -} from "../../tests/claims.ts"; +import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ @@ -17,7 +13,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }); diff --git a/packages/stack/src/services/Pooler.integration.test.ts b/packages/stack/src/services/Pooler.integration.test.ts index b6b5a169dc..e61461fc13 100644 --- a/packages/stack/src/services/Pooler.integration.test.ts +++ b/packages/stack/src/services/Pooler.integration.test.ts @@ -7,11 +7,7 @@ import { ProxyError } from "../Proxy.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { - noContainerClaims, - noDirectoryClaims, - noPublicPortReservations, -} from "../../tests/claims.ts"; +import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ @@ -20,7 +16,6 @@ const options = (root: string) => ({ root, cacheRoot: "/tmp/supabase-stack-artifacts", runtime: "native" as const, - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }); diff --git a/packages/stack/src/services/ProcessRecipe.integration.test.ts b/packages/stack/src/services/ProcessRecipe.integration.test.ts index 3b2f9bc8e5..5b4974efb4 100644 --- a/packages/stack/src/services/ProcessRecipe.integration.test.ts +++ b/packages/stack/src/services/ProcessRecipe.integration.test.ts @@ -48,11 +48,7 @@ import { import type { CatalogOptions, RecipeCreation } from "./Recipe.ts"; import * as Realtime from "./Realtime.ts"; import * as Pooler from "./Pooler.ts"; -import { - noContainerClaims, - noDirectoryClaims, - noPublicPortReservations, -} from "../../tests/claims.ts"; +import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; type TestCreation = RecipeCreation<"rest", Record> & { readonly service: "rest"; @@ -80,7 +76,6 @@ const options: CatalogOptions = { root: "/tmp/process-recipe-test", cacheRoot: "/tmp/process-recipe-test-cache", runtime: "docker", - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }; @@ -465,7 +460,6 @@ const realtimeService = Effect.fn(function* (container: ContainerRuntime) { const recipe = yield* makeProcessRecipe( creation, { - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-test", @@ -964,7 +958,6 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-port-race", @@ -1043,7 +1036,6 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-port-exhaustion", @@ -1185,7 +1177,6 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-deadline", @@ -1252,7 +1243,6 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-unrelated-failure", @@ -1361,7 +1351,6 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-exit-failure", diff --git a/packages/stack/src/services/Realtime.integration.test.ts b/packages/stack/src/services/Realtime.integration.test.ts index 664e9373dd..deaa617bdc 100644 --- a/packages/stack/src/services/Realtime.integration.test.ts +++ b/packages/stack/src/services/Realtime.integration.test.ts @@ -6,11 +6,7 @@ import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { - noContainerClaims, - noDirectoryClaims, - noPublicPortReservations, -} from "../../tests/claims.ts"; +import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ @@ -19,7 +15,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }); diff --git a/packages/stack/src/services/Recipe.ts b/packages/stack/src/services/Recipe.ts index 018cfc2bca..986bf1011b 100644 --- a/packages/stack/src/services/Recipe.ts +++ b/packages/stack/src/services/Recipe.ts @@ -3,7 +3,7 @@ import type { Stream } from "effect"; import type { Effect, Ref } from "effect"; import type { ServiceKind } from "../Artifacts.ts"; import type { ServiceDefinition } from "../Service.ts"; -import type { ContainerClaims, DirectoryClaims } from "../namespace/Claims.ts"; +import type { DirectoryClaims } from "../namespace/Claims.ts"; import type { PortError } from "../Ports.ts"; import type { EngineTarget, HostGateway } from "../runtime/Container.ts"; import type { DockerHelperRegistry } from "../storage/DockerHelperRegistry.ts"; @@ -60,8 +60,6 @@ export interface CatalogOptions { readonly hostGateway?: HostGateway; /** The engine endpoint and identity the owner resolved once at startup; absent when native. */ readonly engineTarget?: EngineTarget; - /** Journals this service's containers before they are created, for the namespace's reconcile loop. */ - readonly containerClaims: ContainerClaims; /** Journals directories a service creates outside its data root (for example under `/tmp`). */ readonly directoryClaims: DirectoryClaims; /** Read-only: true when the per-user registry holds `port` for any stack's public listener. */ diff --git a/packages/stack/src/services/Rest.integration.test.ts b/packages/stack/src/services/Rest.integration.test.ts index 97e3746145..5d3ad83bda 100644 --- a/packages/stack/src/services/Rest.integration.test.ts +++ b/packages/stack/src/services/Rest.integration.test.ts @@ -11,11 +11,7 @@ import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { ProxyError } from "../Proxy.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { - noContainerClaims, - noDirectoryClaims, - noPublicPortReservations, -} from "../../tests/claims.ts"; +import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const makeTestState = (root: string) => @@ -40,7 +36,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }); diff --git a/packages/stack/src/services/Studio.integration.test.ts b/packages/stack/src/services/Studio.integration.test.ts index d114bd9ec9..18f9670382 100644 --- a/packages/stack/src/services/Studio.integration.test.ts +++ b/packages/stack/src/services/Studio.integration.test.ts @@ -3,11 +3,7 @@ import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Layer } from "effect"; import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; -import { - noContainerClaims, - noDirectoryClaims, - noPublicPortReservations, -} from "../../tests/claims.ts"; +import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; const options = (root: string) => ({ stackId: "catalog-studio", @@ -15,7 +11,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }); diff --git a/packages/stack/src/services/Vector.integration.test.ts b/packages/stack/src/services/Vector.integration.test.ts index 44624817f2..2b752c3bd6 100644 --- a/packages/stack/src/services/Vector.integration.test.ts +++ b/packages/stack/src/services/Vector.integration.test.ts @@ -5,11 +5,7 @@ import { Effect, FileSystem, Layer } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; -import { - noContainerClaims, - noDirectoryClaims, - noPublicPortReservations, -} from "../../tests/claims.ts"; +import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string, runtime: "docker" | "native") => ({ @@ -19,7 +15,6 @@ const options = (root: string, runtime: "docker" | "native") => ({ cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, runtime, ...(runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }); diff --git a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts index 57c629a2f9..f44c968ec9 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts @@ -2,7 +2,6 @@ import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { ConfigProvider, - Context, Crypto, Data, Deferred, @@ -22,15 +21,12 @@ import { import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { fileURLToPath } from "node:url"; import { postgresVersion, resolveArtifact } from "../Artifacts.ts"; -import * as Claims from "../namespace/Claims.ts"; import { makeContainerRuntime, type EngineTarget } from "../runtime/Container.ts"; import { makeDatabaseSnapshots } from "../services/DatabaseSnapshot.ts"; -import * as StackNamespace from "../StackNamespace.ts"; import { makeDockerDatabaseStorage } from "./DockerDatabaseStorage.ts"; import { makeDockerHelperRegistry } from "./DockerHelperRegistry.ts"; import { removeTestRunVolumes } from "../../tests/docker-volume-run.ts"; import type { DockerHelperRegistry } from "./DockerHelperRegistry.ts"; -import { noContainerClaims } from "../../tests/claims.ts"; // An unpinned target: these tests exercise storage marker/volume logic, not endpoint pinning. const dockerTarget: EngineTarget = { engine: "docker", argv: [], daemonId: "test-daemon-id" }; @@ -196,7 +192,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { }), ); const storage = yield* makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target: dockerTarget, stackId: "helper-create-cancel", @@ -241,14 +236,12 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const instanceRoot = path.join(storageRoot, "database"); yield* fs.makeDirectory(instanceRoot, { recursive: true }); const container = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root, imageMirrors: () => [helperMirror], }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const storage = yield* makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target: dockerTarget, stackId: "storage-helper-mirror", @@ -276,84 +269,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { ).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); }); - it.live("claims a helper container before creating it, and unclaims it after removal", () => { - // Assigned once `state` exists below; the mock engine calls it synchronously at the exact - // moment the "run" command (the one the claim must precede) fires. - let onCreate: () => Effect.Effect = () => Effect.void; - const engine = fakeHelperEngine(() => onCreate()); - return Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "storage-helper-claims-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const instanceRoot = path.join(storageRoot, "database"); - yield* fs.makeDirectory(instanceRoot, { recursive: true }); - const state = Context.get( - yield* Layer.build(StackNamespace.layer({ root: path.join(root, "state", "stack") })), - StackNamespace.Service, - ); - const stackId = "storage-helper-claims"; - const claims = Claims.forStack(state, stackId).containers; - const claimedAtCreate = yield* Deferred.make>(); - onCreate = () => - state.readClaims(stackId).pipe( - Effect.flatMap((current) => Deferred.succeed(claimedAtCreate, current)), - Effect.asVoid, - Effect.orDie, - ); - // Scoped on its own: the helper is a reusable resource unclaimed only when the - // storage's own scope closes (its removeHelper finalizer), not merely after one - // operation, so that close must happen before the unclaim assertion below. - yield* Effect.scoped( - Effect.gen(function* () { - const container = yield* makeContainerRuntime({ - claims, - target: dockerTarget, - root, - imageMirrors: () => [helperMirror], - }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const storage = yield* makeDockerDatabaseStorage({ - claims, - runtime: "docker", - target: dockerTarget, - stackId, - instanceId: "database", - instanceRoot, - root: storageRoot, - cacheRoot, - fs, - path, - crypto, - container, - spawner, - }); - - yield* storage.prepare("17"); - yield* storage.removeData("17"); - }), - ); - - // No sleeps or polling: the deferred is only ever resolved from inside the mock engine, - // synchronously at the claim-then-create boundary. - const atCreate = yield* Deferred.await(claimedAtCreate); - expect(atCreate).toHaveLength(1); - expect(atCreate[0]).toMatchObject({ kind: "container" }); - const runCommand = engine.commands.find((args) => args[0] === "run"); - const nameIndex = runCommand?.indexOf("--name") ?? -1; - const namedByEngine = nameIndex === -1 ? undefined : runCommand?.[nameIndex + 1]; - expect(namedByEngine).toBeDefined(); - // The journaled identity is exactly the one the engine was told to create, not merely - // some container identity: a mismatch would let reconcile remove the wrong resource. - expect(atCreate[0]?.id).toBe(namedByEngine); - expect(yield* state.readClaims(stackId)).toEqual([]); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); - }); - it.live("starts a shared volume helper with the mirror image selected during preparation", () => { const engine = fakeHelperEngine(undefined, { volumePresent: true }); return Effect.scoped( @@ -399,7 +314,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { marker, ); const container = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root, imageMirrors: () => [helperMirror], @@ -407,7 +321,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const helpers = yield* makeDockerHelperRegistry("mirror-test"); const storage = yield* makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target: dockerTarget, stackId, @@ -451,7 +364,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* fs.makeDirectory(targetRoot, { recursive: true }); yield* fs.makeDirectory(cacheRoot, { recursive: true }); const container = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root, }); @@ -459,7 +371,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const stackId = `storage-pg15-${yield* crypto.randomUUIDv4}`; const makeStorage = (instanceId: string, instanceRoot: string) => makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target: dockerTarget, stackId, @@ -560,7 +471,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* fs.makeDirectory(targetRoot, { recursive: true }); yield* fs.makeDirectory(cacheRoot, { recursive: true }); const container = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root, }); @@ -572,7 +482,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { cacheRootValue: string, ) => makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target: dockerTarget, stackId: "storage-test", @@ -759,7 +668,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const instanceRoot = path.join(storageRoot, "database"); yield* fs.makeDirectory(instanceRoot, { recursive: true }); const container = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root, }); @@ -769,7 +677,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { // This run id overrides the ambient one, so the shared run teardown never sees the volume. yield* Effect.addFinalizer(() => removeTestRunVolumes(testRunId).pipe(Effect.orDie)); const storage = yield* makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target: dockerTarget, stackId, @@ -819,7 +726,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-empty-" }); const instanceRoot = path.join(root, "state", "stack", "data", "empty"); const storage = yield* makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target: dockerTarget, stackId: "storage-empty-test", @@ -859,7 +765,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const markerPath = path.join(instanceRoot, ".supabase-database-storage.json"); const helperImage = yield* postgresImage("17"); const storage = yield* makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target: dockerTarget, stackId: "storage-crash-test", @@ -871,7 +776,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { path, crypto, container: yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root: instanceRoot, }), @@ -1025,13 +929,11 @@ describe("Docker database storage", { timeout: 120_000 }, () => { "chown -R 100:101 /instance/data /instance/.supabase-snapshots; chmod -R 700 /instance/data /instance/.supabase-snapshots", ]); const container = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root, }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const storage = yield* makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target: dockerTarget, stackId: `storage-unmarked-${yield* crypto.randomUUIDv4}`, @@ -1073,7 +975,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* fs.makeDirectory(dataRoot, { recursive: true }); yield* fs.writeFileString(path.join(dataRoot, "PG_VERSION"), "17\n"); const storage = yield* makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target: dockerTarget, stackId: "storage-docker-unmarked", @@ -1085,7 +986,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { path, crypto, container: yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root, }), @@ -1115,7 +1015,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* fs.makeDirectory(instanceRoot, { recursive: true }); yield* fs.makeDirectory(cacheRoot, { recursive: true }); const container = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root, }); @@ -1128,7 +1027,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { Effect.provideService(Scope.Scope, helperScope), ); const storage = yield* makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target: dockerTarget, stackId, @@ -1182,7 +1080,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* fs.makeDirectory(instanceRoot, { recursive: true }); yield* fs.makeDirectory(cacheRoot, { recursive: true }); const container = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root, }); @@ -1213,7 +1110,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { ); const makeStorage = (helpers: DockerHelperRegistry) => makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target: dockerTarget, stackId, @@ -1280,7 +1176,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* fs.makeDirectory(instanceRoot, { recursive: true }); yield* fs.makeDirectory(cacheRoot, { recursive: true }); const container = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root, }); @@ -1288,7 +1183,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { const stackId = `storage-missing-${yield* crypto.randomUUIDv4}`; const makeStorage = () => makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target: dockerTarget, stackId, @@ -1367,7 +1261,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* fs.makeDirectory(instanceRoot, { recursive: true }); yield* fs.makeDirectory(cacheRoot, { recursive: true }); const container = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root, }); @@ -1376,7 +1269,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { // One storage object spans the whole case, the way an Owner keeps one per instance for // the stack's lifetime: its resolved identity stays in memory across the sleep below. const storage = yield* makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target: dockerTarget, stackId, @@ -1454,7 +1346,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* fs.makeDirectory(instanceRoot, { recursive: true }); yield* fs.makeDirectory(cacheRoot, { recursive: true }); const container = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root, }); @@ -1463,7 +1354,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { // One storage object spans the whole case, the way an Owner keeps one per instance for // the stack's lifetime: its resolved identity stays in memory across the sleep below. const storage = yield* makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target: dockerTarget, stackId, @@ -1557,7 +1447,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { '{"version":"17","runtime":"docker","profile":"supabase"}', ); const container = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root, }); @@ -1573,7 +1462,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { "chown -R 100:101 /instance/data; chmod 700 /instance/data", ]); const storage = yield* makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target: dockerTarget, stackId: "storage-host-test", @@ -1677,7 +1565,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* fs.makeDirectory(targetRoot, { recursive: true }); yield* fs.makeDirectory(cacheRoot, { recursive: true }); const container = yield* makeContainerRuntime({ - claims: noContainerClaims, target: dockerTarget, root, }); @@ -1688,7 +1575,6 @@ describe("Docker database storage", { timeout: 120_000 }, () => { cacheRootValue: string, ) => makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target: dockerTarget, stackId: "storage-reopen-test", diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index 8b7b58c295..3775c335a1 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -16,7 +16,6 @@ import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/u import { postgresVersion, resolveArtifact } from "../Artifacts.ts"; import { failureMessage } from "../internal/failure-message.ts"; import { testRunLabelArgs as readTestRunLabelArgs } from "../internal/test-run-label.ts"; -import type * as Claims from "../namespace/Claims.ts"; import * as Publication from "../namespace/Publication.ts"; import type { ContainerRuntime, EngineTarget } from "../runtime/Container.ts"; import { composeProjectFor } from "../runtime/ContainerName.ts"; @@ -136,8 +135,6 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") readonly container: ContainerRuntime | undefined; readonly spawner: ChildProcessSpawnerService["Service"]; readonly helpers?: DockerHelperRegistry; - /** Journals each helper container before it is created, for the namespace's reconcile loop. */ - readonly claims: Claims.ContainerClaims; }): Effect.Effect => Effect.gen(function* () { const markerPath = options.path.join(options.instanceRoot, ".supabase-database-storage.json"); @@ -521,11 +518,6 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") /no such container/iu.test(cause.message) ? Effect.void : Effect.fail(cause), ), ); - // Cleanup authority now belongs entirely to this normal teardown; a crash before - // this point still leaves the claim for the next owner's reconcile loop. - yield* options.claims - .unclaim(current) - .pipe(Effect.mapError((cause) => errorFor("claim", cause))); yield* Ref.set(helperId, undefined); yield* Ref.set(helperImage, undefined); yield* Ref.set(helperCleanupPending, false); @@ -583,15 +575,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") const name = `supabase-db-helper-${token}`; const testRunLabel = yield* testRunLabelArgs(); // Register the deterministic owned name before the remote create starts so an - // interrupted docker run can still be removed by the same scope, and journal it so a - // SIGKILL that skips that scope still leaves it recoverable by the next reconcile. - // Required, the same as launch: a container must never be created with an - // unverifiable daemon identity, since a crashed owner could then leave a claim no - // reconcile can ever trust. - const resolvedDaemonId = options.target.daemonId; - yield* options.claims - .claim(name, resolvedDaemonId) - .pipe(Effect.mapError((cause) => errorFor("claim", cause))); + // interrupted docker run can still be removed by the same scope. yield* Ref.set(helperId, name); yield* Ref.set(helperImage, image); const created = yield* engineCommand([ @@ -643,9 +627,6 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") Effect.catchTag("DockerDatabaseStorageError", (cause) => missingContainer(cause.message) ? Effect.void : Effect.fail(cause), ), - Effect.andThen( - options.claims.unclaim(id).pipe(Effect.mapError((cause) => errorFor("claim", cause))), - ), Effect.asVoid, ); // A container left in `created` cannot exec; remove it and start another. @@ -688,14 +669,6 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") .prepareImage(image) .pipe(Effect.mapError((cause) => errorFor("helper", cause))); const testRunLabel = yield* testRunLabelArgs(); - // Journaled before the create starts, so a SIGKILL leaves this shared helper's identity - // recoverable by the next reconcile instead of leaked. Required, the same as launch: a - // container must never be created with an unverifiable daemon identity, since a crashed - // owner could then leave a claim no reconcile can ever trust. - const resolvedDaemonId = options.target.daemonId; - yield* options.claims - .claim(name, resolvedDaemonId) - .pipe(Effect.mapError((cause) => errorFor("claim", cause))); return yield* Effect.uninterruptible( engineCommand([ "run", diff --git a/packages/stack/tests/claims.ts b/packages/stack/tests/claims.ts index a00913eb61..67d2d6ca6f 100644 --- a/packages/stack/tests/claims.ts +++ b/packages/stack/tests/claims.ts @@ -1,15 +1,11 @@ import { Effect } from "effect"; -import type { ContainerClaims, DirectoryClaims } from "../src/namespace/Claims.ts"; +import type { DirectoryClaims } from "../src/namespace/Claims.ts"; import type { PortError } from "../src/Ports.ts"; /** - * No-op claims for tests that exercise container or directory creation without exercising the - * namespace's reconcile loop itself (covered separately by the namespace's own test suite). + * No-op claims for tests that exercise directory creation without exercising the namespace's + * reconcile loop itself (covered separately by the namespace's own test suite). */ -export const noContainerClaims: ContainerClaims = { - claim: () => Effect.void, - unclaim: () => Effect.void, -}; export const noDirectoryClaims: DirectoryClaims = { claim: () => Effect.void, unclaim: () => Effect.void, diff --git a/packages/stack/tests/docker-fixture.integration.test.ts b/packages/stack/tests/docker-fixture.integration.test.ts index 3117a36eca..8fccf5a18f 100644 --- a/packages/stack/tests/docker-fixture.integration.test.ts +++ b/packages/stack/tests/docker-fixture.integration.test.ts @@ -6,7 +6,7 @@ import { Context, Effect, FileSystem, Layer, Redacted, Schema, Stream } from "ef import { makeStandaloneService } from "./standalone-service.ts"; import { makeDatabase, type BackendEndpoint } from "../src/services/Database.ts"; import { makeDockerDatabaseRoot } from "./docker-fixture.ts"; -import { noContainerClaims, noDirectoryClaims } from "./claims.ts"; +import { noDirectoryClaims } from "./claims.ts"; import { dockerEngineTarget } from "./engine-target.ts"; const marker = Schema.Struct({ backend: Schema.Literal("docker"), volume: Schema.String }); @@ -76,7 +76,6 @@ describe("Docker database fixture isolation", { timeout: 180_000 }, () => { cacheRoot: `${root}/cache`, runtime: "docker", engineTarget: dockerEngineTarget, - containerClaims: noContainerClaims, directoryClaims: noDirectoryClaims, }); const [first, second] = yield* Effect.all([make(firstRoot), make(secondRoot)]); diff --git a/packages/stack/tests/docker-storage-publish-loop-fixture.ts b/packages/stack/tests/docker-storage-publish-loop-fixture.ts index f2a0a36497..dea0d8d925 100644 --- a/packages/stack/tests/docker-storage-publish-loop-fixture.ts +++ b/packages/stack/tests/docker-storage-publish-loop-fixture.ts @@ -3,7 +3,6 @@ import { Console, Crypto, Effect, FileSystem, Layer, Path } from "effect"; import { ChildProcessSpawner } from "effect/unstable/process"; import { makeContainerRuntime } from "../src/runtime/Container.ts"; import { makeDockerDatabaseStorage } from "../src/storage/DockerDatabaseStorage.ts"; -import { noContainerClaims } from "./claims.ts"; const [instanceRoot, root, cacheRoot] = process.argv.slice(2); if (instanceRoot === undefined || root === undefined || cacheRoot === undefined) @@ -38,12 +37,10 @@ const program = Effect.scoped( // marker the caller's `prepare` already published through it. const target = { engine: "docker" as const, argv: [], daemonId: "test-daemon-id" }; const container = yield* makeContainerRuntime({ - claims: noContainerClaims, target, root: instanceRoot, }); const storage = yield* makeDockerDatabaseStorage({ - claims: noContainerClaims, runtime: "docker", target, stackId: "storage-crash-test", From 10584a4d1580f546b33e7d363b3b8a9ca88616fd Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 14:56:01 +0200 Subject: [PATCH 042/142] refactor(stack): drop dead lifecycle state and use a fixed crash-loop cooldown --- packages/stack/ARCHITECTURE.md | 2 +- packages/stack/src/Lifecycle.ts | 101 +++--------- packages/stack/src/Lifecycle.unit.test.ts | 145 ++---------------- .../src/Orchestrator.integration.test.ts | 38 +---- packages/stack/src/Orchestrator.ts | 11 +- packages/stack/src/Service.ts | 4 - 6 files changed, 42 insertions(+), 259 deletions(-) diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 8663e056af..48a438ee5a 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -274,7 +274,7 @@ Admission requires the target and its whole prerequisite closure to be ready. An ### Crash-loop breaker -Each launch failure or unrequested exit counts once per generation. Three consecutive failures open the breaker for 30 seconds, doubling on each reopening up to 5 minutes. While open, admission fails at once naming the last cause; the cooldown's end re-admits one shared recovery attempt. A generation that stays ready for 30 seconds clears the count. An explicit start, restart or composition start resets the breaker. +Each launch failure or unrequested exit counts once per generation. Three consecutive failures open the breaker for a fixed 30 seconds. While open, admission fails at once naming the last cause; the cooldown's end re-admits one shared recovery attempt, and a further failure reopens it for the same 30 seconds. An explicit start, restart or composition start resets the breaker. ### Chart 3: dependency awareness lives in the reducer diff --git a/packages/stack/src/Lifecycle.ts b/packages/stack/src/Lifecycle.ts index ff88493097..cbbf2f8851 100644 --- a/packages/stack/src/Lifecycle.ts +++ b/packages/stack/src/Lifecycle.ts @@ -1,12 +1,10 @@ import { Data } from "effect"; -/** Default per-waiter wake budget; a service's spec may override it for known-slow launches. */ -const defaultWaiterBudgetMillis = 120_000; +/** The wake budget every waiter gets. */ +export const waiterBudgetMillis = 120_000; const waiterCap = 256; const breakerThreshold = 3; -const breakerBaseCooldownMillis = 30_000; -const breakerMaxCooldownMillis = 300_000; -const breakerStabilityMillis = 30_000; +const breakerCooldownMillis = 30_000; /** The pause between a failed re-check of a blocking session and the next one. */ const reprobeSpacingMillis = 1_000; @@ -17,7 +15,6 @@ export interface ServiceSpec { readonly prerequisites: ReadonlyArray; /** Undefined means the service never sleeps on idle. */ readonly idleMillis?: number; - readonly waiterBudgetMillis?: number; } /** @@ -78,7 +75,7 @@ export const makeGraph = (specs: ReadonlyArray): LifecycleGraph => type Intent = "eager" | "lazy" | "stopped"; /** A launch's progress, named so a waiter's budget error can report where it's still blocked. */ -type Stage = "preparing" | "artifacts" | "launching"; +type Stage = "preparing" | "launching"; /** * A service's point in its launch/stop cycle. `Running.ready` tracks the live health signal @@ -97,21 +94,13 @@ const Phase = Data.taggedEnum(); interface BreakerState { readonly consecutiveFailures: number; readonly openUntil: number | undefined; - /** The cooldown this breaker will use next time it opens; doubles on each repeat open. */ - readonly cooldownMillis: number; readonly lastCause: unknown; - /** The generation and timestamp its stability reset is counting from; cleared by a readiness loss or an exit. */ - readonly stableGeneration: number | undefined; - readonly stableSince: number | undefined; } const initialBreaker: BreakerState = { consecutiveFailures: 0, openUntil: undefined, - cooldownMillis: breakerBaseCooldownMillis, lastCause: undefined, - stableGeneration: undefined, - stableSince: undefined, }; interface Waiter { @@ -120,10 +109,11 @@ interface Waiter { readonly deadline: number | undefined; /** False for an acquisition (such as the Functions inspector) that bypasses target readiness. */ readonly requireReady: boolean; - /** Whether admission opens a work lease; an explicit operation's wait only observes readiness. */ - readonly lease: boolean; } +/** Whether admission opens a work lease; an explicit operation's wait only observes readiness. */ +const isLeaseWaiter = (waiter: Waiter): boolean => waiter.deadline !== undefined; + export interface ServiceState { readonly intent: Intent; readonly phase: Phase; @@ -341,7 +331,6 @@ const isActiveForGraphUpdate = (service: ServiceState): boolean => const specsEqual = (a: ServiceSpec, b: ServiceSpec): boolean => a.activation === b.activation && a.idleMillis === b.idleMillis && - a.waiterBudgetMillis === b.waiterBudgetMillis && a.prerequisites.length === b.prerequisites.length && a.prerequisites.every((prerequisite) => b.prerequisites.includes(prerequisite)); @@ -390,24 +379,6 @@ const sessionReady = (state: LifecycleState, id: string): boolean => { ); }; -const isStabilized = (breaker: BreakerState, phase: Phase, now: number): boolean => - breaker.stableGeneration !== undefined && - breaker.stableGeneration === generationOf(phase) && - breaker.stableSince !== undefined && - now - breaker.stableSince >= breakerStabilityMillis; - -/** Bakes in a stability reset that has already been earned before its evidence is discarded. */ -const materializeStability = (breaker: BreakerState, phase: Phase, now: number): BreakerState => - isStabilized(breaker, phase, now) - ? { - ...breaker, - consecutiveFailures: 0, - cooldownMillis: breakerBaseCooldownMillis, - stableGeneration: undefined, - stableSince: undefined, - } - : breaker; - /** * Whether the breaker is currently blocking admission. This is a plain flag, not a `now` * comparison: only the dedicated `CooldownElapsed` event clears it, so a stale or unrelated event @@ -469,7 +440,7 @@ const failExplicitDependentWaiters = ( for (const dependent of state.graph.transitiveDependents.get(id) ?? []) { const service = next.services.get(dependent); if (service === undefined) continue; - const explicit = [...service.waiters.values()].filter((waiter) => !waiter.lease); + const explicit = [...service.waiters.values()].filter((waiter) => !isLeaseWaiter(waiter)); if (explicit.length === 0) continue; const waiters = new Map(service.waiters); for (const waiter of explicit) { @@ -523,10 +494,6 @@ const blocksWaiters = (state: LifecycleState, id: string): boolean => { return false; }; -/** A service's wake budget; a traffic waiter's deadline counts from when the client arrived. */ -export const waiterBudgetMillis = (graph: LifecycleGraph, id: string): number => - graph.services.get(id)?.waiterBudgetMillis ?? defaultWaiterBudgetMillis; - /** Queues or immediately resolves one traffic acquisition or explicit readiness wait. */ const openWaiter = ( state: LifecycleState, @@ -554,20 +521,15 @@ const openWaiter = ( if (isBreakerOpen(service.breaker)) return fail(`${id} circuit breaker is open`, service.breaker.lastCause); // The cap bounds sockets held by traffic; explicit operations are few and already bounded. - const trafficWaiters = [...service.waiters.values()].filter((waiter) => waiter.lease).length; + const trafficWaiters = [...service.waiters.values()].filter(isLeaseWaiter).length; if (traffic && trafficWaiters >= waiterCap) return fail(`${id} has too many waiters`, undefined); - const deadline = traffic ? openedAt + waiterBudgetMillis(state.graph, id) : undefined; + const deadline = traffic ? openedAt + waiterBudgetMillis : undefined; const commands: Array = deadline === undefined ? [] : [LifecycleCommand.ArmWaiterTimeout({ id, waiterId, deadline })]; const queued = setService(state, id, (s) => ({ ...s, lastActivity: traffic ? now : s.lastActivity, - waiters: new Map(s.waiters).set(waiterId, { - id: waiterId, - deadline, - requireReady, - lease: traffic, - }), + waiters: new Map(s.waiters).set(waiterId, { id: waiterId, deadline, requireReady }), })); return [retryBlockedCleanup(queued, id, commands), commands]; }; @@ -646,7 +608,6 @@ const rearmPrerequisites = (state: LifecycleState, id: string): LifecycleState = return next; }; -/** Shared breaker/phase bookkeeping for a launch failure or an unrequested exit. */ /** Counts one generation's launch failure or crash, opening the breaker at the threshold. */ const countFailure = ( service: ServiceState, @@ -655,29 +616,14 @@ const countFailure = ( now: number, commands: Array, ): BreakerState => { - const materialized = materializeStability(service.breaker, service.phase, now); - const consecutiveFailures = materialized.consecutiveFailures + 1; + const consecutiveFailures = service.breaker.consecutiveFailures + 1; if (consecutiveFailures < breakerThreshold) - return { - consecutiveFailures, - openUntil: undefined, - cooldownMillis: materialized.cooldownMillis, - lastCause: cause, - stableGeneration: undefined, - stableSince: undefined, - }; - const openUntil = now + materialized.cooldownMillis; + return { consecutiveFailures, openUntil: undefined, lastCause: cause }; + const openUntil = now + breakerCooldownMillis; commands.push( - LifecycleCommand.ArmCooldownTimer({ id, openUntil, delayMillis: materialized.cooldownMillis }), + LifecycleCommand.ArmCooldownTimer({ id, openUntil, delayMillis: breakerCooldownMillis }), ); - return { - consecutiveFailures, - openUntil, - cooldownMillis: Math.min(breakerMaxCooldownMillis, materialized.cooldownMillis * 2), - lastCause: cause, - stableGeneration: undefined, - stableSince: undefined, - }; + return { consecutiveFailures, openUntil, lastCause: cause }; }; const recordFailure = ( @@ -766,7 +712,6 @@ const applyEvent = ( setService(state, id, (s) => ({ ...s, phase: Phase.Running({ generation, ready: true }), - breaker: { ...s.breaker, stableGeneration: generation, stableSince: now }, readinessFailure: undefined, reprobing: false, })), @@ -855,9 +800,6 @@ const applyEvent = ( ...s, phase: Phase.Stopping({ generation }), idleArmedEpoch: undefined, - // Bake in any 30s stability the just-ending generation already earned, since the next - // failure to check it will compare against a different, unrelated generation. - breaker: materializeStability(s.breaker, s.phase, now), })), commands, ]; @@ -894,11 +836,6 @@ const applyEvent = ( return { ...s, phase: Phase.Running({ generation, ready: false }), - breaker: { - ...materializeStability(s.breaker, s.phase, now), - stableGeneration: undefined, - stableSince: undefined, - }, waiters, readinessFailure: { cause, reprobed: s.reprobing }, reprobing: false, @@ -920,7 +857,6 @@ const applyEvent = ( setService(state, id, (s) => ({ ...s, phase: Phase.Running({ generation, ready: true }), - breaker: { ...s.breaker, stableGeneration: generation, stableSince: now }, readinessFailure: undefined, reprobing: false, })), @@ -1234,7 +1170,10 @@ const settle = ( else admitted.push(waiter.id); } if (admitted.length > 0 || expired.length > 0) { - const leased = admitted.filter((waiterId) => service.waiters.get(waiterId)?.lease).length; + const leased = admitted.filter((waiterId) => { + const waiter = service.waiters.get(waiterId); + return waiter !== undefined && isLeaseWaiter(waiter); + }).length; for (const waiterId of admitted) emitted.push(LifecycleCommand.AdmitConnection({ id, waiterId })); for (const waiterId of expired) diff --git a/packages/stack/src/Lifecycle.unit.test.ts b/packages/stack/src/Lifecycle.unit.test.ts index 907f12ab3c..7247bc8acf 100644 --- a/packages/stack/src/Lifecycle.unit.test.ts +++ b/packages/stack/src/Lifecycle.unit.test.ts @@ -395,7 +395,7 @@ describe("Lifecycle reducer", () => { }); describe("circuit breaker", () => { - it("opens after 3 consecutive failures, fails fast naming the cause, recovers once after the cooldown for several waiters at once, and resets after stable readiness", () => { + it("opens after 3 consecutive failures, fails fast naming the cause, and recovers once after the cooldown for several waiters at once", () => { const graph = makeGraph([lazy("flaky")]); let state = initialState(graph, 0); let commandsByStep: ReadonlyArray>; @@ -455,24 +455,9 @@ describe("Lifecycle reducer", () => { }, ])); expect(tagsOf(commandsByStep[0]).toSorted()).toEqual(["AdmitConnection", "AdmitConnection"]); - - // 30s of stable readiness on the same generation resets the breaker lazily, at the next event. - ({ state, commandsByStep } = run(state, [ - { - event: LifecycleEvent.Exited({ - id: "flaky", - generation: recovery.generation, - cause: "crash", - requested: false, - }), - now: 201 + 30_000 + 10 + 30_000 + 1, - }, - ])); - expect(state.services.get("flaky")?.breaker.consecutiveFailures).toBe(1); - expect(state.services.get("flaky")?.breaker.openUntil).toBeUndefined(); }); - it("doubles the cooldown on a second breaker open and an explicit start resets it", () => { + it("reopens for the same fixed cooldown on a further failure, and an explicit start resets it", () => { const graph = makeGraph([lazy("flaky")]); let state = initialState(graph, 0); @@ -500,7 +485,7 @@ describe("Lifecycle reducer", () => { ])); fail(4, 3 + 30_000); expect(state.services.get("flaky")?.breaker.consecutiveFailures).toBe(4); - expect(state.services.get("flaky")?.breaker.openUntil).toBe(3 + 30_000 + 1 + 60_000); + expect(state.services.get("flaky")?.breaker.openUntil).toBe(3 + 30_000 + 1 + 30_000); ({ state } = run( state, @@ -581,112 +566,6 @@ describe("Lifecycle reducer", () => { expect(stale.commandsByStep[0]).toEqual([]); expect(stale.state).toEqual(state); }); - - it("materializes an already-earned stability reset before a readiness loss discards the evidence", () => { - const graph = makeGraph([lazy("flaky")]); - let state = initialState(graph, 0); - - const fail = (waiterId: number, now: number) => { - ({ state } = run(state, [{ event: open("flaky", waiterId), now }])); - const generation = startingGeneration(state, "flaky"); - ({ state } = run(state, [ - { - event: LifecycleEvent.LaunchFailed({ id: "flaky", generation, cause: "boom" }), - now: now + 1, - }, - ])); - }; - fail(1, 0); - fail(2, 100); - - ({ state } = run(state, [{ event: open("flaky", 3), now: 200 }])); - const generation = startingGeneration(state, "flaky"); - ({ state } = run(state, [ - { event: LifecycleEvent.LaunchSucceeded({ id: "flaky", generation }), now: 201 }, - ])); - - ({ state } = run(state, [ - { - event: LifecycleEvent.ReadinessLost({ id: "flaky", generation, cause: "blip" }), - now: 201 + 30_000 + 1, - }, - ])); - expect(state.services.get("flaky")?.breaker.consecutiveFailures).toBe(0); - - const result = run(state, [ - { - event: LifecycleEvent.Exited({ - id: "flaky", - generation, - cause: "crash", - requested: false, - }), - now: 201 + 30_000 + 2, - }, - ]); - expect(result.state.services.get("flaky")?.breaker.consecutiveFailures).toBe(1); - expect(result.state.services.get("flaky")?.breaker.openUntil).toBeUndefined(); - }); - - it("materializes an already-earned stability reset before an idle stop starts a new generation", () => { - const graph = makeGraph([lazy("flaky")]); - let state = initialState(graph, 0); - - const fail = (waiterId: number, now: number) => { - ({ state } = run(state, [{ event: open("flaky", waiterId), now }])); - const generation = startingGeneration(state, "flaky"); - ({ state } = run(state, [ - { - event: LifecycleEvent.LaunchFailed({ id: "flaky", generation, cause: "boom" }), - now: now + 1, - }, - ])); - }; - fail(1, 0); - fail(2, 100); - - ({ state } = run(state, [{ event: open("flaky", 3), now: 200 }])); - const generation = startingGeneration(state, "flaky"); - ({ state } = run(state, [ - { event: LifecycleEvent.LaunchSucceeded({ id: "flaky", generation }), now: 201 }, - { event: LifecycleEvent.ConnectionClosed({ id: "flaky" }), now: 202 }, - ])); - - ({ state } = run(state, [ - { - event: LifecycleEvent.IdleElapsed({ id: "flaky", generation, epoch: 1 }), - now: 202 + 30_000 + 1_000, - }, - ])); - expect(state.services.get("flaky")?.breaker.consecutiveFailures).toBe(0); - - ({ state } = run(state, [ - { - event: LifecycleEvent.Exited({ - id: "flaky", - generation, - cause: undefined, - requested: true, - }), - now: 202 + 30_000 + 1_000 + 10, - }, - ])); - expect(state.services.get("flaky")?.phase).toEqual({ _tag: "Stopped" }); - - const nextAttempt = run(state, [{ event: open("flaky", 4), now: 1_000_000 }]); - const nextGeneration = startingGeneration(nextAttempt.state, "flaky"); - const result = run(nextAttempt.state, [ - { - event: LifecycleEvent.LaunchFailed({ - id: "flaky", - generation: nextGeneration, - cause: "boom", - }), - now: 1_000_001, - }, - ]); - expect(result.state.services.get("flaky")?.breaker.consecutiveFailures).toBe(1); - }); }); it("caps waiters at 256, lets an expiry fail only that waiter while the shared launch continues, and frees its slot for cancellation", () => { @@ -728,13 +607,13 @@ describe("Lifecycle reducer", () => { }); it("expires a waiter instead of admitting it when its deadline already passed by the time the launch succeeds", () => { - const graph = makeGraph([lazy("api", { waiterBudgetMillis: 100, idleMillis: undefined })]); + const graph = makeGraph([lazy("api", { idleMillis: undefined })]); let state = initialState(graph, 0); ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); const generation = startingGeneration(state, "api"); const result = run(state, [ - { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation }), now: 500 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation }), now: 120_000 }, ]); expect(tagsOf(result.commandsByStep[0])).toEqual(["FailConnection"]); expect(result.state.services.get("api")?.leases).toBe(0); @@ -1131,27 +1010,27 @@ describe("Lifecycle reducer", () => { describe("stage reporting", () => { it("updates the starting stage so a waiter's budget error names it", () => { - let state = initialState(makeGraph([lazy("api", { waiterBudgetMillis: 50 })]), 0); + let state = initialState(makeGraph([lazy("api")]), 0); ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); const generation = startingGeneration(state, "api"); ({ state } = run(state, [ { - event: LifecycleEvent.StageChanged({ id: "api", generation, stage: "artifacts" }), + event: LifecycleEvent.StageChanged({ id: "api", generation, stage: "launching" }), now: 1, }, ])); expect(state.services.get("api")?.phase).toEqual({ _tag: "Starting", generation, - stage: "artifacts", + stage: "launching", }); const expired = run(state, [ { event: LifecycleEvent.WaiterExpired({ id: "api", waiterId: 1 }), now: 100 }, ]); const failure = commandTagged(at(expired.commandsByStep, 0), "FailConnection"); - expect(failure.message).toContain("api to finish artifacts"); + expect(failure.message).toContain("api to finish launching"); }); it("ignores a stage change tagged with a stale generation", () => { @@ -1159,7 +1038,7 @@ describe("Lifecycle reducer", () => { ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); const result = run(state, [ { - event: LifecycleEvent.StageChanged({ id: "api", generation: 99, stage: "artifacts" }), + event: LifecycleEvent.StageChanged({ id: "api", generation: 99, stage: "launching" }), now: 1, }, ]); @@ -1676,7 +1555,7 @@ describe("Lifecycle reducer", () => { }); it("caps only traffic waiters and dates a waiter's budget from the client's arrival", () => { - let state = initialState(makeGraph([lazy("api", { waiterBudgetMillis: 100 })]), 0); + let state = initialState(makeGraph([lazy("api")]), 0); let commandsByStep: ReadonlyArray>; ({ state } = run( state, @@ -1717,6 +1596,6 @@ describe("Lifecycle reducer", () => { now: 50, }, ])); - expect(commandTagged(at(commandsByStep, 1), "ArmWaiterTimeout").deadline).toBe(120); + expect(commandTagged(at(commandsByStep, 1), "ArmWaiterTimeout").deadline).toBe(120_020); }); }); diff --git a/packages/stack/src/Orchestrator.integration.test.ts b/packages/stack/src/Orchestrator.integration.test.ts index 3b4ef1fda6..831c85e110 100644 --- a/packages/stack/src/Orchestrator.integration.test.ts +++ b/packages/stack/src/Orchestrator.integration.test.ts @@ -1464,7 +1464,7 @@ describe("failed and crashed services", () => { ); it.live( - "opens the breaker after three crashes, fails fast naming the cause, doubles the cooldown on reopening, and resets on an explicit start", + "opens the breaker after three crashes, fails fast naming the cause, reopens for the same fixed cooldown after a further crash, and resets on an explicit start", () => Effect.scoped( Effect.gen(function* () { @@ -1504,8 +1504,7 @@ describe("failed and crashed services", () => { yield* serveThenCrash; expect(yield* Ref.get(api.starts)).toHaveLength(4); - // Reopening doubles the cooldown: 30 seconds no longer suffice. - yield* TestClock.adjust("30 seconds"); + // A further failure reopens the breaker for the same fixed cooldown. expect( (yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip)).message, ).toBe("api circuit breaker is open: segfault"); @@ -1569,39 +1568,6 @@ describe("failed and crashed services", () => { ), ); - it.live("forgets earlier crashes once a generation stays healthy for 30 seconds", () => - Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - const crashes = yield* Queue.unbounded>(); - const api = yield* makeInstance(orchestrator, "api", { crash: Queue.take(crashes) }); - yield* orchestrator.configure({ - members: [{ id: "api", activation: "lazy" }], - dependencies: [], - }); - yield* orchestrator.startComposition; - const crash = Effect.gen(function* () { - const down = yield* stopped(api).pipe(Effect.forkChild({ startImmediately: true })); - yield* Queue.offer(crashes, Exit.fail(failure("segfault"))); - yield* Fiber.join(down); - }); - - yield* Effect.scoped(orchestrator.acquire("api")); - yield* crash; - yield* Effect.scoped(orchestrator.acquire("api")); - yield* crash; - yield* Effect.scoped(orchestrator.acquire("api")); - yield* TestClock.adjust("30 seconds"); - yield* crash; - - // Without the reset this would be the third consecutive failure, opening the breaker. - yield* Effect.scoped(orchestrator.acquire("api")); - expect(yield* Ref.get(api.starts)).toHaveLength(4); - yield* orchestrator.stopNamespace; - }), - ).pipe(Effect.provide(TestClock.layer())), - ); - it.live("relaunches a crashed prerequisite without replacing its healthy dependent", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/Orchestrator.ts b/packages/stack/src/Orchestrator.ts index 544d4296ba..444e2d8f6b 100644 --- a/packages/stack/src/Orchestrator.ts +++ b/packages/stack/src/Orchestrator.ts @@ -137,6 +137,7 @@ export const CompositionConfig = Schema.Struct({ export interface Status extends ServiceObservation { readonly lifecycle: "stopped" | "starting" | "running" | "stopping"; readonly health: "starting" | "healthy" | "unhealthy" | undefined; + readonly cleanupError: ServiceError | undefined; /** The live generation, if any. */ readonly launchId: number | undefined; /** Whether demand (traffic, a dependent or eager intent) relaunches the service. */ @@ -343,6 +344,10 @@ const statusOf = ( ...execution, lifecycle: lifecycleOf(state, id), health, + cleanupError: + service?.cleanupFailure === undefined + ? undefined + : cleanupError(id, service.cleanupFailure.cause), launchId: phase === undefined || phase._tag === "Stopped" || phase._tag === "Failed" ? undefined @@ -630,7 +635,6 @@ export const make = Effect.fn("Orchestrator.make")(function* @@ -651,7 +655,7 @@ export const make = Effect.fn("Orchestrator.make")(function* } >, openedAt: number, - graph: LifecycleGraph, bounded: Wait | undefined, restore: (effect: Effect.Effect) => Effect.Effect, ) { @@ -707,7 +710,7 @@ export const make = Effect.fn("Orchestrator.make")(function* Effect.fail( new ServiceError({ diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index 7cc957d96a..a3463240f4 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -24,7 +24,6 @@ export interface ServiceObservation { readonly id: string; readonly config: Config; readonly error: ServiceError | undefined; - readonly cleanupError: ServiceError | undefined; readonly exit: Exit.Exit | undefined; readonly currentOperation: ServiceOperation | undefined; readonly registered: boolean; @@ -172,7 +171,6 @@ export const makeService = ( id, config: options.config, error: undefined, - cleanupError: undefined, exit: undefined, currentOperation: undefined, registered: true, @@ -184,7 +182,6 @@ export const makeService = ( SubscriptionRef.update(observations, (value) => ({ ...value, error: value.error ?? error, - cleanupError: error, })); /** Halts and removes one session, retrying only the steps a previous attempt didn't finish. */ @@ -208,7 +205,6 @@ export const makeService = ( } yield* Scope.close(record.scope, Exit.void); yield* Ref.update(current, (value) => (value === record ? undefined : value)); - yield* update({ cleanupError: undefined }); }); /** Runs one readiness program in the session's health scope; interruption yields `undefined`. */ From 46525d0a8f99ac17084e4463383c97fd580db1b1 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 14:50:37 +0200 Subject: [PATCH 043/142] refactor(stack): drop the redundant lease interlock and share the transient-read retry --- packages/stack/src/namespace/Capabilities.ts | 11 + packages/stack/src/namespace/Lease.ts | 78 ++----- .../namespace/Namespace.integration.test.ts | 210 ------------------ packages/stack/src/namespace/Registry.ts | 21 +- 4 files changed, 37 insertions(+), 283 deletions(-) diff --git a/packages/stack/src/namespace/Capabilities.ts b/packages/stack/src/namespace/Capabilities.ts index c464e73614..0e6b2c5795 100644 --- a/packages/stack/src/namespace/Capabilities.ts +++ b/packages/stack/src/namespace/Capabilities.ts @@ -37,3 +37,14 @@ export const transientRetrySchedule = Schedule.exponential("10 millis", 2).pipe( ), Schedule.upTo({ times: 12 }), ); + +/** Retries a read against `transientRetrySchedule` while it hits a sharing violation on `platform`. */ +export const retryTransientRead = + (platform?: NodeJS.Platform) => + (effect: Effect.Effect): Effect.Effect => + effect.pipe( + Effect.retry({ + schedule: transientRetrySchedule, + while: isSharingViolation(platform ?? process.platform), + }), + ); diff --git a/packages/stack/src/namespace/Lease.ts b/packages/stack/src/namespace/Lease.ts index dbed876d42..162425f815 100644 --- a/packages/stack/src/namespace/Lease.ts +++ b/packages/stack/src/namespace/Lease.ts @@ -1,11 +1,6 @@ -import { Effect, Exit, FileSystem, Option, Path, Schedule, Schema, Scope, Semaphore } from "effect"; +import { Effect, Exit, FileSystem, Option, Path, Schedule, Schema, Scope } from "effect"; import type { DatabaseSync } from "node:sqlite"; -import { - isSharingViolation, - namespaceError, - transientRetrySchedule, - type NamespaceError, -} from "./Capabilities.ts"; +import { namespaceError, retryTransientRead, type NamespaceError } from "./Capabilities.ts"; import * as Publication from "./Publication.ts"; import { removeEmptyDirectory } from "./drivers/FileSystem.ts"; import { acquireLock, errcode, isBusy, isMissing, takeLock } from "./drivers/Sqlite.ts"; @@ -58,6 +53,9 @@ export interface Interface { readonly ownerLog: (id: string) => string; } +export const OWNER_FILE = "owner.json"; +export const OWNER_LOG_FILE = "owner.log"; + const checkId = (id: string): Effect.Effect => /^[a-zA-Z0-9_-]+$/u.test(id) ? Effect.void @@ -80,11 +78,9 @@ export const make = ( const path = yield* Path.Path; const stackRoot = (id: string) => path.join(options.root, id); const leasePath = (id: string) => path.join(stackRoot(id), "owner.lock"); - const ownerPath = (id: string) => path.join(stackRoot(id), "owner.json"); - const ownerLog = (id: string) => path.join(stackRoot(id), "owner.log"); - const sharingViolation = isSharingViolation(options.platform ?? process.platform); - const retryTransientRead = (effect: Effect.Effect) => - effect.pipe(Effect.retry({ schedule: transientRetrySchedule, while: sharingViolation })); + const ownerPath = (id: string) => path.join(stackRoot(id), OWNER_FILE); + const ownerLog = (id: string) => path.join(stackRoot(id), OWNER_LOG_FILE); + const retryRead = retryTransientRead(options.platform); /** An open file that was unlinked: SQLite IOERR_VNODE on macOS, IOERR_FSTAT on Linux. */ const isMoved = (error: NamespaceError) => @@ -110,10 +106,6 @@ export const make = ( const scope = yield* Scope.Scope; let held = false; let unlinked = false; - let closed = false; - // Shared by the mutations and the finalizer below, so a publish or retract already in - // flight finishes before the finalizer marks the handle closed and releases the connection. - const gate = yield* Semaphore.make(1); yield* Effect.addFinalizer(() => held ? (unlinked ? Effect.void : removeLeaseFile(id)).pipe( @@ -178,17 +170,6 @@ export const make = ( attemptScope, removeLeaseFile(id).pipe(Effect.map((removed) => (unlinked = removed))), ); - // Registered last so it runs first: taking the gate here waits for a publish or - // retract already in flight, so no mutation can resume after the connection below - // (registered earlier, so it releases after this) is gone. - yield* Scope.addFinalizer( - attemptScope, - gate.withPermit( - Effect.sync(() => { - closed = true; - }), - ), - ); return { ownerLog: ownerLog(id) }; } yield* Scope.close(attemptScope, Exit.void); @@ -210,26 +191,6 @@ export const make = ( }), ), ); - // A handle plucked out of its scope (for example, stored past the scope that acquired it) - // must not mutate another holder's live record once that scope has closed; checking under - // the same gate the finalizer takes rules out a mutation that is already past this check - // resuming after the finalizer has closed the handle and released the connection. Waiting - // for the gate stays interruptible, but once acquired, `effect` runs uninterruptibly: a - // filesystem callback that is already in flight (for example `fs.rm`'s) has no cancellation - // of its own, so interrupting mid-flight would free the gate while that work is still - // pending, letting a later holder's record publish before this removal finally lands on it. - const guarded = (effect: Effect.Effect) => - gate.withPermit( - Effect.uninterruptible( - Effect.suspend(() => - closed - ? Effect.fail( - namespaceError("lease", `The lease handle for stack ${id} has already closed`), - ) - : effect, - ), - ), - ); return { stackId: id, ownerLog: outcome.ownerLog, @@ -240,19 +201,16 @@ export const make = ( // The lease already guarantees a single writer, so an atomic rename over any existing // record (a live one, or a crashed holder's stale one) is enough: a reader never sees a // gap where the file briefly doesn't exist, unlike a prior remove followed by a publish. - yield* guarded( - Publication.publish(fs, path, { - target: ownerPath(id), - content: serialized, - platform: options.platform, - }), - ); + yield* Publication.publish(fs, path, { + target: ownerPath(id), + content: serialized, + platform: options.platform, + }); }), - retractHolder: guarded( - fs - .remove(ownerPath(id), { force: true }) - .pipe(Effect.mapError((cause) => namespaceError("remove", cause))), - ).pipe(Effect.withSpan("Namespace.Lease.retractHolder")), + retractHolder: fs.remove(ownerPath(id), { force: true }).pipe( + Effect.mapError((cause) => namespaceError("remove", cause)), + Effect.withSpan("Namespace.Lease.retractHolder"), + ), } satisfies LeaseHandle; }); @@ -277,7 +235,7 @@ export const make = ( const target = ownerPath(id); // The holder may retract its record at any moment, so a missing record is not an error. const text = yield* fs.readFileString(target).pipe( - retryTransientRead, + retryRead, Effect.map(Option.some), Effect.catchIf( (error) => error.reason._tag === "NotFound", diff --git a/packages/stack/src/namespace/Namespace.integration.test.ts b/packages/stack/src/namespace/Namespace.integration.test.ts index ef12c9c6c8..69e5017713 100644 --- a/packages/stack/src/namespace/Namespace.integration.test.ts +++ b/packages/stack/src/namespace/Namespace.integration.test.ts @@ -584,216 +584,6 @@ describe("stack owner lease", () => { ), ); - it.live("rejects mutations from a handle whose scope already closed", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lease-expired-" }); - const state = yield* makeTestState(root); - const firstScope = yield* Scope.make(); - const expired = yield* state.acquireLease("recycled").pipe(Scope.provide(firstScope)); - yield* Scope.close(firstScope, Exit.void); - - const secondScope = yield* Scope.make(); - const live = yield* state.acquireLease("recycled").pipe(Scope.provide(secondScope)); - yield* live.publishHolder({ - role: "sweeper", - pid: process.pid, - startedAt: "2026-01-01T00:00:00.000Z", - }); - - const publishRejected = yield* expired - .publishHolder({ - role: "sweeper", - pid: process.pid, - startedAt: "2026-01-01T00:00:00.000Z", - }) - .pipe(Effect.flip); - expect(publishRejected).toBeInstanceOf(StackNamespace.NamespaceError); - const retractRejected = yield* expired.retractHolder.pipe(Effect.flip); - expect(retractRejected).toBeInstanceOf(StackNamespace.NamespaceError); - - expect(yield* state.readHolder("recycled")).toEqual({ - role: "sweeper", - pid: process.pid, - startedAt: "2026-01-01T00:00:00.000Z", - }); - yield* Scope.close(secondScope, Exit.void); - }), - ), - ); - - it.live( - "keeps a fresh holder's record intact against a publish already in flight when its scope closes", - () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const id = "racing"; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lease-race-" }); - const ownerJson = path.join(root, id, "owner.json"); - // A's publish pauses here, already past the closed-handle guard, so the scope-closing - // finalizer below can only proceed once A's publish has fully released the gate. - const paused = yield* Deferred.make(); - const entered = yield* Deferred.make(); - const renameCalls = yield* Ref.make(0); - const pausingFs = Layer.effect( - FileSystem.FileSystem, - Effect.sync(() => ({ - ...fs, - rename: (oldPath: string, newPath: string) => - Effect.gen(function* () { - const isFirst = - newPath === ownerJson && - (yield* Ref.getAndUpdate(renameCalls, (n) => n + 1)) === 0; - if (isFirst) { - yield* Deferred.succeed(entered, undefined); - yield* Deferred.await(paused); - } - return yield* fs.rename(oldPath, newPath); - }), - })), - ); - const state = Context.get( - yield* Layer.build(StackNamespace.layer({ root }).pipe(Layer.provide(pausingFs))), - StackNamespace.Service, - ); - - const scopeA = yield* Scope.make(); - const a = yield* state.acquireLease(id).pipe(Scope.provide(scopeA)); - const recordA: StackNamespace.LeaseHolder = { - role: "sweeper", - pid: process.pid, - startedAt: "2026-01-01T00:00:00.000Z", - }; - const publishingA = yield* a.publishHolder(recordA).pipe(Effect.forkScoped); - yield* Deferred.await(entered); - - const closingA = yield* Scope.close(scopeA, Exit.void).pipe(Effect.forkScoped); - const recordB: StackNamespace.LeaseHolder = { - role: "sweeper", - pid: process.pid + 1, - startedAt: "2026-01-01T00:00:01.000Z", - }; - const acquiringB = yield* Effect.scoped( - Effect.gen(function* () { - const b = yield* state.acquireLease(id); - yield* b.publishHolder(recordB); - }), - ).pipe(Effect.forkScoped); - - // Neither A's close nor B's acquisition can complete while A's publish is paused. - yield* Effect.sleep("50 millis"); - expect(closingA.pollUnsafe()).toBeUndefined(); - expect(acquiringB.pollUnsafe()).toBeUndefined(); - - yield* Deferred.succeed(paused, undefined); - yield* Fiber.join(publishingA); - yield* Fiber.join(closingA); - yield* Fiber.join(acquiringB); - - expect(yield* state.readHolder(id)).toEqual(recordB); - }), - ), - ); - - it.live( - "keeps a fresh holder's record intact against an interrupted removal whose native callback is still pending", - () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const id = "racing-interrupt"; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "namespace-lease-race-interrupt-", - }); - const ownerJson = path.join(root, id, "owner.json"); - const entered = yield* Deferred.make(); - const fire = yield* Deferred.make(); - // Settles once the background removal below has actually run, independent of whether - // anything is still listening for its `resume` call: proves the real filesystem effect, - // not just the Effect-level interrupt, has finished before the test checks final state. - const settled = yield* Deferred.make(); - // Only A's own removal (the first call) is held pending; B's later, unrelated calls on - // the same path (its own stale-record cleanup, then its own eventual retraction) must - // pass straight through, or they would deadlock on the very same native callback. - let intercepted = false; - const context = yield* Effect.context(); - const runFork = Effect.runForkWith(context); - // Mirrors `Effect.effectify(fs.rm)`: no cleanup effect is returned, so interrupting the - // fiber waiting on this callback only stops listening for it. The real removal, once - // started in the background below, keeps running and completes on its own regardless. - const callbackFs = Layer.succeed(FileSystem.FileSystem, { - ...fs, - remove: (file: string, options?: Parameters[1]) => - file === ownerJson && !intercepted - ? Effect.callback((resume) => { - intercepted = true; - Deferred.doneUnsafe(entered, Exit.void); - runFork( - Deferred.await(fire).pipe( - Effect.andThen(fs.remove(file, options)), - Effect.matchEffect({ - onSuccess: () => Effect.sync(() => resume(Effect.void)), - onFailure: (cause) => Effect.sync(() => resume(Effect.fail(cause))), - }), - Effect.ensuring(Effect.sync(() => Deferred.doneUnsafe(settled, Exit.void))), - ), - ); - }) - : fs.remove(file, options), - }); - const state = Context.get( - yield* Layer.build(StackNamespace.layer({ root }).pipe(Layer.provide(callbackFs))), - StackNamespace.Service, - ); - - const scopeA = yield* Effect.acquireRelease(Scope.make(), (s) => - Scope.close(s, Exit.void), - ); - const a = yield* state.acquireLease(id).pipe(Scope.provide(scopeA)); - - // Runs this one fiber with scheduler yields prevented, so it executes synchronously up - // to the intercepted callback above: by the time this call returns, `entered` is - // already done, which is what makes the steps below a deterministic barrier instead of - // a timing guess. - const mutationContext = Context.add(context, Scheduler.PreventSchedulerYield, true); - const retracting = Effect.runForkWith(mutationContext)(a.retractHolder); - yield* Deferred.await(entered); - yield* Effect.acquireUseRelease( - Effect.void, - () => - Effect.sync(() => { - retracting.interruptUnsafe(); - // Without `Effect.uninterruptible` around the guarded mutation, this interrupt - // unwinds synchronously right here, releasing the gate before the native removal - // below ever runs; with it, the fiber stays suspended until the release phase - // lets the callback fire. - expect(retracting.pollUnsafe()).toBeUndefined(); - }), - () => - Deferred.succeed(fire, undefined).pipe( - Effect.andThen(Fiber.await(retracting)), - Effect.andThen(Deferred.await(settled)), - ), - ); - yield* Scope.close(scopeA, Exit.void); - - const recordB: StackNamespace.LeaseHolder = { - role: "sweeper", - pid: process.pid + 1, - startedAt: "2026-01-01T00:00:01.000Z", - }; - const b = yield* state.acquireLease(id); - yield* b.publishHolder(recordB); - - expect(yield* state.readHolder(id)).toEqual(recordB); - }), - ), - ); - it.live("reports a free lease without creating a lease file", () => run( Effect.gen(function* () { diff --git a/packages/stack/src/namespace/Registry.ts b/packages/stack/src/namespace/Registry.ts index b2f5c04988..1698b6afef 100644 --- a/packages/stack/src/namespace/Registry.ts +++ b/packages/stack/src/namespace/Registry.ts @@ -1,13 +1,9 @@ import { Effect, Exit, FileSystem, Path, Schedule, Schema, Scope } from "effect"; import { CompositionConfig } from "../Orchestrator.ts"; import { ServiceCreation } from "../services/Catalog.ts"; -import { - isSharingViolation, - namespaceError, - transientRetrySchedule, - type NamespaceError, -} from "./Capabilities.ts"; +import { namespaceError, retryTransientRead, type NamespaceError } from "./Capabilities.ts"; import { CLAIMS_FILE } from "./Claims.ts"; +import { OWNER_FILE, OWNER_LOG_FILE } from "./Lease.ts"; import * as Publication from "./Publication.ts"; import { removeEmptyDirectory } from "./drivers/FileSystem.ts"; import { acquireLock, isBusy, takeLock } from "./drivers/Sqlite.ts"; @@ -136,18 +132,17 @@ export const make = ( const stackRoot = (id: string) => path.join(root, id); const statePath = (id: string) => path.join(stackRoot(id), "state.json"); - const sharingViolation = isSharingViolation(options.platform ?? process.platform); - const retryTransientRead = (effect: Effect.Effect) => + const retryRead = (effect: Effect.Effect) => effect.pipe( - Effect.retry({ schedule: transientRetrySchedule, while: sharingViolation }), + retryTransientRead(options.platform), Effect.mapError((cause) => namespaceError("read", cause)), ); const read = Effect.fn("Namespace.Registry.read")(function* (id: string) { yield* checkId(id); const target = statePath(id); - const exists = yield* fs.exists(target).pipe(retryTransientRead); + const exists = yield* fs.exists(target).pipe(retryRead); if (!exists) return undefined; - const text = yield* fs.readFileString(target).pipe(retryTransientRead); + const text = yield* fs.readFileString(target).pipe(retryRead); const state = yield* decodeState(text, id, target); if (state.id !== id) return yield* namespaceError("identity", "State document identity does not match its path"); @@ -192,8 +187,8 @@ export const make = ( yield* checkId(id); for (const file of [ statePath(id), - path.join(stackRoot(id), "owner.json"), - path.join(stackRoot(id), "owner.log"), + path.join(stackRoot(id), OWNER_FILE), + path.join(stackRoot(id), OWNER_LOG_FILE), path.join(stackRoot(id), CLAIMS_FILE), ]) yield* fs From 0a51dcde5837cd8700a371d33617eafc5fd683d3 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 15:10:07 +0200 Subject: [PATCH 044/142] refactor(cli): list changed keys and the recreate command when a saved stack rejects a config change --- apps/cli/src/command-internal/stack-config.ts | 61 +- .../stack/destroy/destroy.handler.ts | 18 +- .../experimental/stack/stack.shared.ts | 18 + .../experimental/stack/start/SIDE_EFFECTS.md | 15 +- .../experimental/stack/start/start.handler.ts | 470 ++---------- .../stack/start/start.integration.test.ts | 726 +----------------- .../stack/status/status.handler.ts | 14 +- 7 files changed, 125 insertions(+), 1197 deletions(-) diff --git a/apps/cli/src/command-internal/stack-config.ts b/apps/cli/src/command-internal/stack-config.ts index bf8ddcd138..b5acd04dcf 100644 --- a/apps/cli/src/command-internal/stack-config.ts +++ b/apps/cli/src/command-internal/stack-config.ts @@ -162,18 +162,9 @@ const envNestedPortOrConfigured = ( return typeof nested?.[key] === "number" ? configured : undefined; }; -/** A setting's `config.toml` key and the `SUPABASE_*` env var that overrides it. */ -export interface StackEndpointSetting { - readonly configPath: string; +/** A port setting's `config.toml` section/key and the `SUPABASE_*` env var that overrides it. */ +interface PortSetting { readonly envVar: string; -} - -/** - * A port setting's `config.toml` key and env var, shared by `createCreations` and - * `stackEndpointSetting` so both report the same names; a new endpoint still needs an entry in - * `endpointSettingsByServiceEndpoint`. - */ -interface PortSetting extends StackEndpointSetting { readonly section: string; readonly nestedSection?: string; readonly key: string; @@ -183,62 +174,52 @@ const DB_PORT: PortSetting = { envVar: "SUPABASE_DB_PORT", section: "db", key: "port", - configPath: "db.port", }; const API_PORT: PortSetting = { envVar: "SUPABASE_API_PORT", section: "api", key: "port", - configPath: "api.port", }; const STUDIO_PORT: PortSetting = { envVar: "SUPABASE_STUDIO_PORT", section: "studio", key: "port", - configPath: "studio.port", }; const DB_POOLER_PORT: PortSetting = { envVar: "SUPABASE_DB_POOLER_PORT", section: "db", nestedSection: "pooler", key: "port", - configPath: "db.pooler.port", }; const LOCAL_SMTP_PORT: PortSetting = { envVar: "SUPABASE_LOCAL_SMTP_PORT", section: "local_smtp", key: "port", - configPath: "local_smtp.port", }; const LOCAL_SMTP_SMTP_PORT: PortSetting = { envVar: "SUPABASE_LOCAL_SMTP_SMTP_PORT", section: "local_smtp", key: "smtp_port", - configPath: "local_smtp.smtp_port", }; const LOCAL_SMTP_POP3_PORT: PortSetting = { envVar: "SUPABASE_LOCAL_SMTP_POP3_PORT", section: "local_smtp", key: "pop3_port", - configPath: "local_smtp.pop3_port", }; const ANALYTICS_PORT: PortSetting = { envVar: "SUPABASE_ANALYTICS_PORT", section: "analytics", key: "port", - configPath: "analytics.port", }; const ANALYTICS_VECTOR_PORT: PortSetting = { envVar: "SUPABASE_ANALYTICS_VECTOR_PORT", section: "analytics", key: "vector_port", - configPath: "analytics.vector_port", }; const EDGE_RUNTIME_INSPECTOR_PORT: PortSetting = { envVar: "SUPABASE_EDGE_RUNTIME_INSPECTOR_PORT", section: "edge_runtime", key: "inspector_port", - configPath: "edge_runtime.inspector_port", }; /** Resolves one `PortSetting` against the loaded document and env, picking the nested variant when needed. */ @@ -260,44 +241,6 @@ const resolvePort = ( env, ); -/** - * Maps a saved stack endpoint (service + endpoint name) to the `PortSetting` that controls it. - * An endpoint missing here (e.g. `pooler.http`, `realtime.rpc`) is always automatic. - */ -const endpointSettingsByServiceEndpoint: Readonly> = { - "database.sql": DB_PORT, - "pooler.sql": DB_POOLER_PORT, - "analytics.http": ANALYTICS_PORT, - "vector.http": ANALYTICS_VECTOR_PORT, - "studio.http": STUDIO_PORT, - "mail.http": LOCAL_SMTP_PORT, - "mail.smtp": LOCAL_SMTP_SMTP_PORT, - "mail.pop3": LOCAL_SMTP_POP3_PORT, - "functions.inspector": EDGE_RUNTIME_INSPECTOR_PORT, - "rest.http": API_PORT, - "auth.http": API_PORT, - "realtime.http": API_PORT, - "storage.http": API_PORT, - "functions.http": API_PORT, -}; - -/** The config.toml key and env var override for a service endpoint, when the CLI exposes one. */ -export const stackEndpointSetting = ( - service: string, - endpoint: string, -): StackEndpointSetting | undefined => endpointSettingsByServiceEndpoint[`${service}.${endpoint}`]; - -/** - * `db.major_version`'s config key and `SUPABASE_DB_MAJOR_VERSION` override. Unlike the ports - * above, `envOverrideMajorVersion` (shared with `db-bootstrap` and the legacy local stack) hardcodes - * its own name/field, so there is no single call site to read this from without widening that - * shared helper's signature; the two literals here are kept in sync by hand. - */ -export const stackMajorVersionSetting: StackEndpointSetting = { - configPath: "db.major_version", - envVar: "SUPABASE_DB_MAJOR_VERSION", -}; - const authProviderNames = [ "apple", "azure", diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts index b26f30bc22..936f36acee 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts @@ -8,23 +8,15 @@ import { CommandSettings } from "../../../../config/command-settings.service.ts" import { TelemetryState } from "../../../../telemetry/telemetry-state.service.ts"; import { StackApi, - StackTargetError, rejectStackOutput, skippedRuntimeCleanupWarning, StackTargetResolver, + mapTargetError, validateStackTarget, } from "../stack.shared.ts"; import type { StackDestroyFlags } from "./destroy.command.ts"; import { StackCommandDestroyError } from "./destroy.errors.ts"; -const mapTargetError = (error: StackTargetError) => - new StackCommandDestroyError({ - reason: error.reason, - message: error.message, - ...(error.suggestion === undefined ? {} : { suggestion: error.suggestion }), - cause: error, - }); - const destroyError = (cause: StackError) => new StackCommandDestroyError({ reason: "unknown", @@ -41,11 +33,13 @@ export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( const settings = yield* CommandSettings; const api = yield* StackApi; const outputFlag = yield* Effect.serviceOption(OutputFlag); - yield* rejectStackOutput(outputFlag).pipe(Effect.mapError(mapTargetError)); + yield* rejectStackOutput(outputFlag).pipe( + Effect.mapError(mapTargetError((props) => new StackCommandDestroyError(props))), + ); yield* validateStackTarget({ stack: Option.getOrUndefined(flags.stack), stackId: Option.getOrUndefined(flags.stackId), - }).pipe(Effect.mapError(mapTargetError)); + }).pipe(Effect.mapError(mapTargetError((props) => new StackCommandDestroyError(props)))); const resolver = yield* StackTargetResolver; const path = yield* Path.Path; @@ -56,7 +50,7 @@ export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( ...(Option.isSome(flags.stackId) ? { id: flags.stackId.value } : {}), runtime: "auto", }) - .pipe(Effect.mapError(mapTargetError)); + .pipe(Effect.mapError(mapTargetError((props) => new StackCommandDestroyError(props)))); if (target.id === undefined) return yield* new StackCommandDestroyError({ reason: "flags", diff --git a/apps/cli/src/commands/experimental/stack/stack.shared.ts b/apps/cli/src/commands/experimental/stack/stack.shared.ts index 49db2ca5f0..fc082e8b2d 100644 --- a/apps/cli/src/commands/experimental/stack/stack.shared.ts +++ b/apps/cli/src/commands/experimental/stack/stack.shared.ts @@ -44,6 +44,24 @@ export class StackTargetError extends Data.TaggedError("ExperimentalStackTargetE } } +/** Rewraps a `StackTargetError` into a command's own error type, preserving reason, message, and suggestion. */ +export const mapTargetError = + ( + make: (props: { + readonly reason: StackTargetError["reason"]; + readonly message: string; + readonly suggestion?: string; + readonly cause: StackTargetError; + }) => E, + ) => + (error: StackTargetError): E => + make({ + reason: error.reason, + message: error.message, + ...(error.suggestion === undefined ? {} : { suggestion: error.suggestion }), + cause: error, + }); + interface StackTargetResolverShape { readonly resolve: (input: { readonly projectRoot: string; diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index 78113174b1..a6799883c5 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -154,13 +154,8 @@ example. Failures retain typed command errors and package diagnostics. Telemetry after success or failure. A rejected configuration change additionally carries `stack_changes` on the JSON/stream-json error -envelope: one entry per affected service/path pair (a shared setting such as the API port appears -once per API-backed service even though the text message deduplicates it to one line; a service -with more than one incompatible path, such as a changed endpoint alongside an artifact version -change, gets one entry per path), each with `service`, `path` (the composition planner's dotted -path, e.g. `endpoints.http.port`, not a `config.toml` key — an added or dropped endpoint instead -reports `endpoints.`, since the planner only reports the bare `endpoints` path for that -case), `key`, `saved`, `requested`, and `editable`. `recreate_command` is the exact `supabase stack -destroy --stack-id ` invocation, without `--yes`, since destroy deletes local database data; -running it non-interactively or with `--output-format json`/`--output-format stream-json` requires -passing `--yes` explicitly. +envelope: one entry per affected service/path pair, each with `service`, `path` (the composition +planner's dotted path, e.g. `endpoints.http.port`), `saved`, and `requested`. `recreate_command` is +the exact `supabase stack destroy --stack-id ` invocation, without `--yes`, since destroy +deletes local database data; running it non-interactively or with `--output-format +json`/`--output-format stream-json` requires passing `--yes` explicitly. diff --git a/apps/cli/src/commands/experimental/stack/start/start.handler.ts b/apps/cli/src/commands/experimental/stack/start/start.handler.ts index f263ff634d..1c36f221f2 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.handler.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.handler.ts @@ -1,4 +1,4 @@ -import { catalogPins, defaultRuntime, postgresVersion } from "@supabase/stack/internal/artifacts"; +import { defaultRuntime } from "@supabase/stack/internal/artifacts"; import { connectionEnv, renderStackSummary, @@ -49,19 +49,13 @@ import { seedBucketsRun, } from "../../../../command-internal/seed-buckets.ts"; import { loadLocalProjectContext } from "../../../../command-internal/local-project-context.ts"; -import { - loadStackConfig, - stackEndpointSetting, - stackMajorVersionSetting, - type StackEndpointSetting, -} from "../../../../command-internal/stack-config.ts"; -import { envOverride } from "../../../../command-internal/local-config-values.ts"; +import { loadStackConfig } from "../../../../command-internal/stack-config.ts"; import { StackApi, stackCapabilityForService, - StackTargetError, StackTargetResolver, failedOutcomesDetail, + mapTargetError, rejectStackOutput, validateStackTarget, } from "../stack.shared.ts"; @@ -93,14 +87,6 @@ const validateExclusions = (exclusions: ReadonlyArray) => { ); }; -const mapTargetError = (error: StackTargetError) => - new StackCommandStartError({ - reason: error.reason, - message: error.message, - ...(error.suggestion === undefined ? {} : { suggestion: error.suggestion }), - cause: error, - }); - const stackError = ( cause: { readonly message: string } & Partial>, members: ReadonlyArray<{ readonly id: string; readonly service: string }> = [], @@ -174,9 +160,8 @@ const sameKinds = ( /** * Whether this set of requested services would run Studio without the REST API it depends on. - * The single source of truth for that dependency: both the real `--exclude` guard and the - * incompatible-change suggestion builder (which proposes a candidate exclusion set) validate - * against it, so a suggested `--exclude` can never itself fail this check. + * The single source of truth for that dependency: the real `--exclude` guard validates against + * it to reject a request that would otherwise leave Studio stranded. */ const studioNeedsRest = (requestedServices: ReadonlyArray<{ readonly service: string }>): boolean => requestedServices.some(({ service }) => service === "studio") && @@ -185,272 +170,65 @@ const studioNeedsRest = (requestedServices: ReadonlyArray<{ readonly service: st const isRecord = (value: unknown): value is Readonly> => typeof value === "object" && value !== null; -const endpointPortLabel = (endpoints: unknown, name: string): string => { - const intent = isRecord(endpoints) ? endpoints[name] : undefined; - const port = isRecord(intent) ? intent.port : undefined; - return port === "auto" ? "automatic" : typeof port === "number" ? String(port) : "unset"; -}; - -const endpointNamesOf = (endpoints: unknown): ReadonlyArray => - isRecord(endpoints) ? Object.keys(endpoints) : []; - -const databaseVersionOf = ( - creation: ServiceCreation | ServiceCreationInput | undefined, -): string | undefined => (creation?.service === "database" ? creation.config.version : undefined); - -const majorVersionOf = (version: string): string => version.split(".")[0] ?? version; - -/** - * The catalog's default pinned version for a service — what `resolveArtifact` selects when a - * creation's own top-level `version` is unset, which every CLI-built creation leaves unset. - */ -const defaultArtifactVersion = (service: PlannedInstance["service"]): string | undefined => - catalogPins().find((pin) => pin.service === service && pin.isDefault)?.pin.upstreamVersion; - -/** Renders a dotted config key as its `config.toml` section/key pair, e.g. `[db] major_version`. */ -const formatConfigPath = (path: string): string => { - const segments = path.split("."); - const key = segments.pop(); - return `[${segments.join(".")}] ${key}`; -}; - -/** The display key for a setting: its env var when that's what overrides it, else its config key. */ -const settingKeyLabel = ( - setting: StackEndpointSetting, - projectEnvValues: Readonly>, -): string => - envOverride(setting.envVar, undefined, projectEnvValues) !== undefined - ? setting.envVar - : formatConfigPath(setting.configPath); - -/** - * One incompatible path, reported as the JSON/stream-json error envelope's `stack_changes` - * entries (contract documented in `SIDE_EFFECTS.md`): one entry per affected service/path pair. - * `editable` marks whether `key` is a `config.toml` key or env var the user can revert, or plain - * wording for a catalog-pinned artifact, Postgres build, or endpoint presence change. - */ -interface StructuredSettingChange { +/** One changed path within an incompatible saved-stack member, as reported in `stack_changes`. */ +interface IncompatibleChange { readonly service: string; readonly path: string; - readonly key: string; readonly saved: string; readonly requested: string; - readonly editable: boolean; -} - -/** - * Why a change is non-editable, so the suggestion builder can pick accurate wording instead of - * re-deriving it from `StructuredSettingChange`'s display strings. Not part of the public - * `stack_changes` contract — only `change` below is. - */ -type ChangeReason = "setting" | "artifact" | "pinned-artifact" | "endpoint-presence"; - -interface DescribedChange { - readonly reason: ChangeReason; - readonly change: StructuredSettingChange; } -const settingChange = ( - reason: ChangeReason, - service: PlannedInstance["service"], - path: string, - key: string, - saved: string, - requested: string, - editable: boolean, -): DescribedChange => ({ reason, change: { service, path, key, saved, requested, editable } }); - -/** Whether a named endpoint is present at all, not what its port is. */ -const endpointPresenceChange = ( - service: PlannedInstance["service"], - name: string, - savedCreation: ServiceCreation | undefined, - requestedCreation: ServiceCreationInput | undefined, -): DescribedChange => { - const savedExposed = endpointNamesOf(savedCreation?.endpoints).includes(name); - const requestedExposed = endpointNamesOf(requestedCreation?.endpoints).includes(name); - return settingChange( - "endpoint-presence", - service, - `endpoints.${name}`, - `${service} ${name} endpoint`, - savedExposed ? "exposed" : "not exposed", - requestedExposed ? "exposed" : "not exposed", - false, - ); -}; - -/** - * Every endpoint added to, or dropped from, the request. The planner's bare `endpoints` path only - * says the two endpoint sets differ (e.g. a saved composition created with no public endpoint at - * all); this names exactly which endpoint(s) and in which direction, one entry per endpoint. - */ -const endpointPresenceChanges = ( - service: PlannedInstance["service"], - savedCreation: ServiceCreation | undefined, - requestedCreation: ServiceCreationInput | undefined, -): ReadonlyArray => { - const savedNames = endpointNamesOf(savedCreation?.endpoints); - const requestedNames = endpointNamesOf(requestedCreation?.endpoints); - const differing = [...new Set([...savedNames, ...requestedNames])].filter( - (name) => savedNames.includes(name) !== requestedNames.includes(name), - ); - return differing.map((name) => - endpointPresenceChange(service, name, savedCreation, requestedCreation), - ); -}; - -const describeSettingChange = ( - service: PlannedInstance["service"], - path: string, - savedCreation: ServiceCreation | undefined, - requestedCreation: ServiceCreationInput | undefined, - projectEnvValues: Readonly>, -): ReadonlyArray => { - if (service === "database" && path === "config.version") { - // `postgresVersion` resolves a bare major alias (e.g. "17") to the pinned build the - // composition plan actually compared, so the saved/requested pair reflects what changed. - const savedVersion = postgresVersion(databaseVersionOf(savedCreation) ?? "unknown"); - const requestedVersion = postgresVersion(databaseVersionOf(requestedCreation) ?? "unknown"); - const savedMajor = majorVersionOf(savedVersion); - const requestedMajor = majorVersionOf(requestedVersion); - // Same major but different pinned build: `major_version` doesn't control this, so reverting - // it wouldn't fix anything — name the actual (unpinnable) versions instead. - if (savedMajor === requestedMajor) - return [ - settingChange( - "artifact", - service, - path, - "Postgres build", - savedVersion, - requestedVersion, - false, - ), - ]; - return [ - settingChange( - "setting", - service, - path, - settingKeyLabel(stackMajorVersionSetting, projectEnvValues), - savedMajor, - requestedMajor, - true, - ), - ]; - } - // The planner reports the bare `endpoints` path when one side has no endpoints object at all - // (e.g. a saved composition created with no public endpoint): name which endpoint(s) the - // request adds or drops rather than misreporting this as an artifact/version change. - if (path === "endpoints") { - const presence = endpointPresenceChanges(service, savedCreation, requestedCreation); - if (presence.length > 0) return presence; - // The planner only reports this path when the endpoint sets genuinely differ; kept as a - // defensive fallback in case a future endpoint shape diffs without changing key sets. - return [ - settingChange("artifact", service, path, `${service} endpoints`, "changed", "changed", false), - ]; - } - if (path.startsWith("endpoints.")) { - const [endpointName, ...rest] = path.split(".").slice(1); - if (endpointName === undefined) - return [ - settingChange("artifact", service, path, `${service} ${path}`, "changed", "changed", false), - ]; - if (rest.length === 0) - // A two-segment `endpoints.` path (unlike the three-segment `endpoints..port` - // a port-value change produces) means the planner only found one side's endpoint object at - // all — this endpoint's presence differs, not its port. - return [endpointPresenceChange(service, endpointName, savedCreation, requestedCreation)]; - const setting = stackEndpointSetting(service, endpointName); - if (setting !== undefined) - return [ - settingChange( - "setting", - service, - path, - settingKeyLabel(setting, projectEnvValues), - endpointPortLabel(savedCreation?.endpoints, endpointName), - endpointPortLabel(requestedCreation?.endpoints, endpointName), - true, - ), - ]; - // A changed endpoint value with no config.toml key or env var override (e.g. `pooler.http`, - // `realtime.rpc`, always automatic): name it plainly instead of implying an editable setting. - return [ - settingChange("artifact", service, path, `${service} ${path}`, "changed", "changed", false), - ]; - } - if (path === "version") { - // CLI-built creations never set a top-level `version`, relying on the catalog's default: - // resolve that default for an unset side instead of reporting the raw (always-unset) field. - const defaultVersion = defaultArtifactVersion(service); - const savedVersion = savedCreation?.version ?? defaultVersion ?? "unknown"; - const requestedVersion = requestedCreation?.version ?? defaultVersion ?? "unknown"; - // The saved stack's explicit pin happens to equal what this release would select by default: - // there's no real artifact difference, so don't blame "this CLI release". - const reason: ChangeReason = savedVersion === requestedVersion ? "pinned-artifact" : "artifact"; - return [ - settingChange( - reason, - service, - path, - `${service} artifact version`, - savedVersion, - requestedVersion, - false, - ), - ]; - } - // Unreachable: the planner's own incompatible-path filter only yields "version", "endpoints", - // "endpoints.", or (database) "config.version". Kept for exhaustiveness. - return [ - settingChange("artifact", service, path, `${service} ${path}`, "changed", "changed", false), - ]; +/** Reads the raw value a dotted planner path (e.g. `config.version`) points to on a creation. */ +const valueAt = (creation: unknown, path: string): unknown => + path + .split(".") + .reduce((value, segment) => (isRecord(value) ? value[segment] : undefined), creation); + +const describeValue = (value: unknown): string => { + if (value === undefined) return "unset"; + if (value === "auto") return "automatic"; + if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") + return String(value); + return JSON.stringify(value); }; -/** Every incompatible path across every rejected saved member, as one structured list. */ -const incompatibleSettingChanges = ( +/** Every incompatible path across every rejected saved member, as one flat list. */ +const incompatibleChanges = ( planned: ReadonlyArray, savedConfigById: ReadonlyMap, requested: ReadonlyArray, - projectEnvValues: Readonly>, -): ReadonlyArray => +): ReadonlyArray => planned .filter((entry) => entry.member && entry.change === "incompatible") .flatMap((entry) => // Narrowed by the filter above; `Extract` isn't inferred through `.filter`. entry.change === "incompatible" - ? entry.paths.flatMap((path) => - describeSettingChange( - entry.service, + ? entry.paths.map((path): IncompatibleChange => { + const savedCreation = savedConfigById.get(entry.id); + const requestedCreation = requested.find( + (creation) => creation.service === entry.service, + ); + return { + service: entry.service, path, - savedConfigById.get(entry.id), - requested.find((creation) => creation.service === entry.service), - projectEnvValues, - ), - ) + saved: describeValue(valueAt(savedCreation, path)), + requested: describeValue(valueAt(requestedCreation, path)), + }; + }) : [], ); -/** One deduplicated text line per distinct setting change (shared API port lines collapse to one). */ -const settingChangeLines = ( - changes: ReadonlyArray, -): ReadonlyArray => { - const seen = new Set(); - const lines: Array = []; - for (const change of changes) { - const line = `${change.key}: saved ${change.saved}, requested ${change.requested}`; - if (seen.has(line)) continue; - seen.add(line); - lines.push(line); - } - return lines; -}; - const dedupe = (values: ReadonlyArray): ReadonlyArray => [...new Set(values)]; +/** One deduplicated text line per distinct changed path (a shared API port collapses to one). */ +const changeLines = (changes: ReadonlyArray): ReadonlyArray => + dedupe( + changes.map( + (change) => + `${change.service} ${change.path}: saved ${change.saved}, requested ${change.requested}`, + ), + ); + /** * The exact `supabase stack destroy` invocation that recreates this stack. Always targets * `--stack-id`: a `--stack ` destroy re-resolves the name against the caller's current @@ -459,164 +237,30 @@ const dedupe = (values: ReadonlyArray): ReadonlyArray => [...new */ const destroyCommandFor = (id: string): string => `supabase stack destroy --stack-id ${id}`; -/** The revert clause for the editable keys among a rejection's changes, or `undefined` for none. */ -const revertAdvice = (editableKeys: ReadonlyArray): string | undefined => - editableKeys.length === 0 - ? undefined - : editableKeys.length === 1 - ? `Revert ${editableKeys[0]} to its saved value` - : "Revert the settings listed to their saved values"; - -/** Whether every one of these services' capabilities can be excluded with `--exclude`. */ -const allExcludable = (services: ReadonlyArray): boolean => - services.length > 0 && - services.every((service) => - (STACK_START_EXCLUDABLE_CAPABILITIES as ReadonlyArray).includes( - stackCapabilityForService(service), - ), - ); - -/** - * Grows a candidate `--exclude` set until requesting the remaining capabilities would actually - * start (validated with `studioNeedsRest`, the same check the real flag goes through), or gives - * up once a dependency can't itself be excluded. Bounded by the number of excludable capabilities, - * since each iteration adds at least one and today's only dependency (Studio needs REST) can't - * cycle back on itself. - */ -const resolveExclusionCapabilities = ( - creations: ReadonlyArray, - initialExclusions: ReadonlyArray, - candidateCapabilities: ReadonlyArray, -): ReadonlyArray | undefined => { - let capabilities = candidateCapabilities; - for (let attempt = 0; attempt <= STACK_START_EXCLUDABLE_CAPABILITIES.length; attempt++) { - const remaining = selectedCreations(creations, [...initialExclusions, ...capabilities]); - if (!studioNeedsRest(remaining)) return capabilities; - // Today's only capability dependency: Studio needs REST. Bail if Studio is already in the - // set (adding it again wouldn't help) or somehow isn't itself excludable. - if ( - capabilities.includes("studio") || - !(STACK_START_EXCLUDABLE_CAPABILITIES as ReadonlyArray).includes("studio") - ) - return undefined; - capabilities = dedupe([...capabilities, "studio"]); - } - return undefined; -}; - -/** Rejects every saved member whose endpoints or artifact versions the request would change. */ +/** Rejects every saved member whose composition the request would incompatibly change. */ const incompatibleChange = ( planned: ReadonlyArray, savedConfigById: ReadonlyMap, requested: ReadonlyArray, - projectEnvValues: Readonly>, stackIdentity: { readonly id: string; readonly name?: string }, - creations: ReadonlyArray, - currentExclusions: ReadonlyArray, ): | { readonly error: StackCommandStartError; - readonly changes: ReadonlyArray; + readonly changes: ReadonlyArray; readonly command: string; } | undefined => { - const described = incompatibleSettingChanges( - planned, - savedConfigById, - requested, - projectEnvValues, - ); - if (described.length === 0) return undefined; - const changes = described.map(({ change }) => change); - const lines = settingChangeLines(changes); + const changes = incompatibleChanges(planned, savedConfigById, requested); + if (changes.length === 0) return undefined; const command = destroyCommandFor(stackIdentity.id); const nameNote = stackIdentity.name === undefined ? "" : ` (stack ${stackIdentity.name})`; - const destroyClause = `\`${command}\`${nameNote} to recreate the stack — this permanently deletes its local database data.`; - - const settingKeys = dedupe( - described.filter((d) => d.reason === "setting").map((d) => d.change.key), - ); - const artifactKeys = dedupe( - described.filter((d) => d.reason === "artifact").map((d) => d.change.key), - ); - const pinnedKeys = dedupe( - described.filter((d) => d.reason === "pinned-artifact").map((d) => d.change.key), - ); - const endpointChanges = described.filter((d) => d.reason === "endpoint-presence"); - const addedEndpoints = endpointChanges.filter((d) => d.change.requested === "exposed"); - const removedEndpoints = endpointChanges.filter((d) => d.change.saved === "exposed"); - const addedEndpointKeys = dedupe(addedEndpoints.map((d) => d.change.key)); - const removedEndpointKeys = dedupe(removedEndpoints.map((d) => d.change.key)); - - const nonEditable = - artifactKeys.length > 0 || pinnedKeys.length > 0 || endpointChanges.length > 0; - const revert = revertAdvice(settingKeys); - - // Purely editable changes (ports, major_version): the original simple two-way choice. - if (!nonEditable && revert !== undefined) - return { - changes, - command, - error: new StackCommandStartError({ - reason: "invalid-config", - message: `The saved stack cannot adopt these changes: ${lines.join("; ")}`, - suggestion: `${revert} to keep the stack and its data, or run ${destroyClause}`, - }), - }; - - const sentences: Array = []; - if (artifactKeys.length > 0) - sentences.push( - `This CLI release starts a different ${artifactKeys.join(" and ")} than the saved stack.`, - ); - if (pinnedKeys.length > 0) - sentences.push( - `The saved stack pins an explicit ${pinnedKeys.join(" and ")} that this release would select by default anyway.`, - ); - if (addedEndpointKeys.length > 0) - sentences.push( - `This request needs the ${addedEndpointKeys.join(", ")} that the saved stack doesn't have.`, - ); - if (removedEndpointKeys.length > 0) - sentences.push( - `The saved stack has the ${removedEndpointKeys.join(", ")} that this request no longer needs.`, - ); - - // Requesting a new endpoint the saved stack never had is sometimes avoidable by excluding the - // whole capability instead of destroying the stack — but only when every incompatible change - // (not just the endpoint ones) belongs to a service this exclusion would drop from the request; - // an unrelated incompatibility elsewhere (e.g. a database major-version change) would still - // reject start regardless, so the exclusion would be offered without actually fixing anything. - const addedServices = dedupe(addedEndpoints.map((d) => d.change.service)) as ReadonlyArray< - PlannedInstance["service"] - >; - const excludableAddedServices = allExcludable(addedServices) - ? new Set(addedServices) - : undefined; - const canExcludeInstead = - excludableAddedServices !== undefined && - described.every((d) => excludableAddedServices.has(d.change.service)); - // A candidate exclusion that would itself fail to start (e.g. excluding REST while Studio stays - // requested) is resolved by pulling in a dependent capability if one is excludable, or dropped. - const resolvedCapabilities = canExcludeInstead - ? resolveExclusionCapabilities( - creations, - currentExclusions, - dedupe(addedServices.map(stackCapabilityForService)), - ) - : undefined; - const exclusionClause = - resolvedCapabilities !== undefined - ? ` Run the command again with \`--exclude ${resolvedCapabilities.join(",")}\` to open it without requesting ${resolvedCapabilities.length === 1 ? "that capability" : "those capabilities"}, or run ${destroyClause}` - : ` Run ${destroyClause}`; - return { changes, command, error: new StackCommandStartError({ reason: "invalid-config", - message: `The saved stack cannot adopt these changes: ${lines.join("; ")}`, - suggestion: `${sentences.join(" ")}${exclusionClause}`, + message: `The saved stack cannot adopt these changes: ${changeLines(changes).join("; ")}`, + suggestion: `Run \`${command}\`${nameNote} to recreate the stack — this permanently deletes its local database data.`, }), }; }; @@ -667,12 +311,14 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags const outputFlag = yield* Effect.serviceOption(OutputFlag); const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - yield* rejectStackOutput(outputFlag).pipe(Effect.mapError(mapTargetError)); + yield* rejectStackOutput(outputFlag).pipe( + Effect.mapError(mapTargetError((props) => new StackCommandStartError(props))), + ); const exclusions = yield* validateExclusions(flags.exclude); yield* validateStackTarget({ stack: Option.getOrUndefined(flags.stack), stackId: Option.getOrUndefined(flags.stackId), - }).pipe(Effect.mapError(mapTargetError)); + }).pipe(Effect.mapError(mapTargetError((props) => new StackCommandStartError(props)))); const target = yield* resolver .resolve({ projectRoot: settings.workdir, @@ -680,7 +326,7 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags ...(Option.isSome(flags.stackId) ? { id: flags.stackId.value } : {}), runtime: flags.runtime, }) - .pipe(Effect.mapError(mapTargetError)); + .pipe(Effect.mapError(mapTargetError((props) => new StackCommandStartError(props)))); const runtime = yield* RuntimeInfo; const selectedRuntime = yield* selectStackRuntime(target.runtime).pipe( Effect.mapError( @@ -957,15 +603,7 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags ...(target.name === undefined ? {} : { name: target.name }), }; const savedConfigById = new Map(currentStatuses.map(({ id, config: saved }) => [id, saved])); - const rejected = incompatibleChange( - planned, - savedConfigById, - requested, - config.projectEnvValues, - stackIdentity, - creations, - exclusions, - ); + const rejected = incompatibleChange(planned, savedConfigById, requested, stackIdentity); if (rejected !== undefined) { const machineErrorContext = yield* Effect.serviceOption(MachineErrorContext); if (Option.isSome(machineErrorContext)) diff --git a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts index bea7a31fdb..e1dbe2351b 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts @@ -19,7 +19,7 @@ import { DEFAULT_LOCAL_JWT_SECRET, DEFAULT_POSTGRES_ROOT_KEY, } from "@supabase/stack/defaults"; -import { catalogPins, postgresVersion } from "@supabase/stack/internal/artifacts"; +import { postgresVersion } from "@supabase/stack/internal/artifacts"; import { StackError, type ServiceCreation, @@ -1233,7 +1233,28 @@ describe("experimental stack start", () => { }).pipe(Effect.provide(BunServices.layer)), ); - it.live("names the config key and both values when a saved port changes", () => + // The planner normalizes an automatic shared API port to the saved fixed one. + it.live( + "accepts a shared API port going from fixed back to automatic, reusing the saved port", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-api-to-auto-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "api-to-auto"\n[api]\nport = 54321\n', + ); + const fixture = fakeStack(); + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + + yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "api-to-auto"\n'); + yield* fixture.stack.composition.stop; + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live("names the changed key and the destroy command when a saved port changes", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-port-config-" }); @@ -1257,25 +1278,28 @@ describe("experimental stack start", () => { expect(error).toMatchObject({ reason: "invalid-config", - message: expect.stringContaining("[api] port: saved 54321, requested 54999"), + message: expect.stringContaining("saved 54321, requested 54999"), + suggestion: expect.stringContaining( + `supabase stack destroy --stack-id ${fixture.stack.id}`, + ), }); // The rejection leaves the stopped composition untouched: no recompose was attempted. expect(fixture.composed).toBe(1); }).pipe(Effect.provide(BunServices.layer)), ); - it.live("names a dedicated (non-shared) port's own config key", () => + it.live("collects changes across multiple saved members into one message", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-dedicated-port-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-multi-change-" }); yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "dedicated-port"\n'); + yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "multi-change"\n'); const fixture = fakeStack(); yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); yield* fs.writeFileString( `${root}/supabase/config.toml`, - 'project_id = "dedicated-port"\n[studio]\nport = 12345\n', + 'project_id = "multi-change"\n[db]\nmajor_version = 15\n[studio]\nport = 12345\n', ); yield* fixture.stack.composition.stop; const error = yield* stackStart(flags()).pipe( @@ -1285,276 +1309,17 @@ describe("experimental stack start", () => { expect(error).toMatchObject({ reason: "invalid-config", - message: expect.stringContaining("[studio] port: saved automatic, requested 12345"), + message: expect.stringContaining("saved 17, requested 15"), }); - // A dedicated port only affects its own service, unlike the shared API port. - expect(error).toBeInstanceOf(StackCommandStartError); - if (error instanceof StackCommandStartError) - expect(error.message).not.toContain("[api] port"); - }).pipe(Effect.provide(BunServices.layer)), - ); - - // The production planner (`fixedApiPorts`/`withSharedApiPort`) normalizes a requested - // automatic shared-API port to the composition's already-fixed value whenever one exists, so a - // saved fixed port going back to automatic in `config.toml` reuses the saved port rather than - // failing. This locks down that non-obvious compatible case: it is not an incompatible path. - it.live( - "accepts a shared API port going from fixed back to automatic, reusing the saved port", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-api-to-auto-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "api-to-auto"\n[api]\nport = 54321\n', - ); - const fixture = fakeStack(); - yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); - - yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "api-to-auto"\n'); - yield* fixture.stack.composition.stop; - // Does not throw: the planner treats this as compatible (`change: "unchanged"`), not an - // incompatible path to report. Reusing the already-bound port for the resumed instance is - // `packages/stack`'s own concern, not asserted here. - yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); - }).pipe(Effect.provide(BunServices.layer)), - ); - - it.live( - "collects simultaneous database-version and port changes into one error with plural revert wording", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-multi-change-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "multi-change"\n'); - const fixture = fakeStack(); - yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); - - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "multi-change"\n[db]\nmajor_version = 15\n[studio]\nport = 12345\n', - ); - yield* fixture.stack.composition.stop; - const error = yield* stackStart(flags()).pipe( - Effect.provide(layers(root, fixture)), - Effect.flip, - ); - - expect(error).toMatchObject({ - reason: "invalid-config", - message: expect.stringContaining("[db] major_version: saved 17, requested 15"), - }); - expect(error).toMatchObject({ - message: expect.stringContaining("[studio] port: saved automatic, requested 12345"), - }); - expect(error).toBeInstanceOf(StackCommandStartError); - if (error instanceof StackCommandStartError) - expect(error.suggestion).toContain( - "Revert the settings listed to their saved values to keep the stack and its data", - ); - }).pipe(Effect.provide(BunServices.layer)), - ); - - it.live("names the env var override when SUPABASE_*_PORT set the saved port", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-port-env-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "port-env"\n'); - const fixture = fakeStack(); - yield* withEnvVar( - "SUPABASE_API_PORT", - "54321", - stackStart(flags()).pipe(Effect.provide(layers(root, fixture))), - ); - - yield* fixture.stack.composition.stop; - const error = yield* withEnvVar( - "SUPABASE_API_PORT", - "54999", - stackStart(flags()).pipe(Effect.provide(layers(root, fixture)), Effect.flip), - ); - expect(error).toMatchObject({ - reason: "invalid-config", - message: expect.stringContaining("SUPABASE_API_PORT: saved 54321, requested 54999"), + message: expect.stringContaining("saved automatic, requested 12345"), }); + expect(error).toBeInstanceOf(StackCommandStartError); + if (error instanceof StackCommandStartError) + expect(error.suggestion).toContain(`supabase stack destroy --stack-id ${fixture.stack.id}`); }).pipe(Effect.provide(BunServices.layer)), ); - it.live( - "names [db] major_version and the destroy command when the saved Postgres version changes", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-major-version-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "major-version"\n'); - const fixture = fakeStack(); - yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); - - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "major-version"\n[db]\nmajor_version = 15\n', - ); - yield* fixture.stack.composition.stop; - const error = yield* stackStart(flags()).pipe( - Effect.provide(layers(root, fixture)), - Effect.flip, - ); - - expect(error).toMatchObject({ - reason: "invalid-config", - message: expect.stringContaining("[db] major_version: saved 17, requested 15"), - suggestion: expect.stringContaining( - `Revert [db] major_version to its saved value to keep the stack and its data, or run \`supabase stack destroy --stack-id ${fixture.stack.id}\` to recreate the stack`, - ), - }); - expect(error).toMatchObject({ suggestion: expect.stringContaining("database data") }); - }).pipe(Effect.provide(BunServices.layer)), - ); - - it.live( - "names the full Postgres build, not major_version, when only the pinned build differs", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-pg-build-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "pg-build"\n[db]\nmajor_version = 17\n', - ); - const fixture = fakeStack(); - yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); - const database = fixture.members.find(({ service }) => service === "database"); - if (database?.service !== "database") return yield* Effect.die("Database missing"); - const observed = yield* database.status; - if (observed.config.service !== "database") - return yield* Effect.die("Database config missing"); - const pinnedVersion = postgresVersion("17"); - // A saved build the current catalog no longer pins (`postgresVersion` only normalizes a - // recognized alias): same major as the requested `17`, different full build. - yield* database.restart({ - config: { ...observed.config.config, version: "17.0.0-stale-build" }, - }); - yield* fixture.stack.composition.stop; - const error = yield* stackStart(flags()).pipe( - Effect.provide(layers(root, fixture)), - Effect.flip, - ); - - expect(error).toMatchObject({ - reason: "invalid-config", - message: expect.stringContaining( - `Postgres build: saved 17.0.0-stale-build, requested ${pinnedVersion}`, - ), - }); - expect(error).toBeInstanceOf(StackCommandStartError); - if (error instanceof StackCommandStartError) { - expect(error.message).not.toContain("major_version"); - expect(error.suggestion).not.toContain("Revert"); - expect(error.suggestion).toContain( - "This CLI release starts a different Postgres build than the saved stack.", - ); - expect(error.suggestion).toContain( - `supabase stack destroy --stack-id ${fixture.stack.id}`, - ); - } - }).pipe(Effect.provide(BunServices.layer)), - ); - - it.live( - "drops the revert sentence for an artifact-version-only mismatch and explains the fix in plain language", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-artifact-version-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "artifact-version"\n', - ); - const fixture = fakeStack(); - yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); - const rest = fixture.members.find(({ service }) => service === "rest"); - if (rest?.service !== "rest") return yield* Effect.die("REST missing"); - const restObserved = yield* rest.status; - if (restObserved.config.service !== "rest") return yield* Effect.die("REST config missing"); - yield* rest.restart({ ...restObserved.config, version: "rest-v1-stale" }); - yield* fixture.stack.composition.stop; - const error = yield* stackStart(flags()).pipe( - Effect.provide(layers(root, fixture)), - Effect.flip, - ); - - expect(error).toMatchObject({ - reason: "invalid-config", - message: expect.stringContaining("rest artifact version"), - }); - expect(error).toBeInstanceOf(StackCommandStartError); - if (error instanceof StackCommandStartError) { - expect(error.suggestion).not.toContain("Revert"); - expect(error.suggestion).toContain( - "This CLI release starts a different rest artifact version than the saved stack.", - ); - expect(error.suggestion).toContain( - `supabase stack destroy --stack-id ${fixture.stack.id}`, - ); - } - }).pipe(Effect.provide(BunServices.layer)), - ); - - it.live( - "uses the destroy-only wording when a non-editable change accompanies an editable one", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-mixed-editable-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "mixed-editable"\n', - ); - const fixture = fakeStack(); - yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); - const rest = fixture.members.find(({ service }) => service === "rest"); - if (rest?.service !== "rest") return yield* Effect.die("REST missing"); - const restObserved = yield* rest.status; - if (restObserved.config.service !== "rest") return yield* Effect.die("REST config missing"); - yield* rest.restart({ ...restObserved.config, version: "rest-v1-stale" }); - yield* fixture.stack.composition.stop; - - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "mixed-editable"\n[studio]\nport = 12345\n', - ); - const error = yield* stackStart(flags()).pipe( - Effect.provide(layers(root, fixture)), - Effect.flip, - ); - - expect(error).toMatchObject({ - reason: "invalid-config", - // The editable change still appears in the message even though reverting it alone - // can't unblock start: the non-editable artifact-version change still would. - message: expect.stringContaining("[studio] port: saved automatic, requested 12345"), - }); - expect(error).toBeInstanceOf(StackCommandStartError); - if (error instanceof StackCommandStartError) { - expect(error.suggestion).not.toContain("Revert"); - expect(error.suggestion).toContain( - "This CLI release starts a different rest artifact version than the saved stack.", - ); - expect(error.suggestion).toContain( - `supabase stack destroy --stack-id ${fixture.stack.id}`, - ); - } - }).pipe(Effect.provide(BunServices.layer)), - ); - it.live("emits structured stack_changes and recreate_command on the JSON error envelope", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -1577,16 +1342,14 @@ describe("experimental stack start", () => { expect(envelope._tag).toBe("Error"); expect(envelope.error).toMatchObject({ code: "ExperimentalStackStartError", - message: expect.stringContaining("[db] major_version: saved 17, requested 15"), + message: expect.stringContaining("saved 17, requested 15"), }); expect(envelope.stack_changes).toEqual([ { service: "database", path: "config.version", - key: "[db] major_version", saved: "17", requested: "15", - editable: true, }, ]); expect(envelope.recreate_command).toBe( @@ -1595,421 +1358,6 @@ describe("experimental stack start", () => { }).pipe(Effect.provide(BunServices.layer)), ); - it.live( - "reports one stack_changes entry per affected service while collapsing the text to one line", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-start-shared-port-dedup-", - }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "shared-port-dedup"\n[api]\nport = 54321\n', - ); - const fixture = fakeStack(); - yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); - // The shared API port is bound by every API-backed service the default config enables; - // this only exercises the dedup/per-entry contract if more than one of them is present. - const apiBackedServices = fixture.members - .map(({ service }) => service) - .filter((service) => - ["rest", "auth", "realtime", "storage", "functions"].includes(service), - ); - expect(apiBackedServices.length).toBeGreaterThan(1); - - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "shared-port-dedup"\n[api]\nport = 54999\n', - ); - yield* fixture.stack.composition.stop; - const { layer, stdio } = jsonErrorLayers(root, fixture, "json"); - yield* stackStart(flags()).pipe(withJsonErrorHandling, Effect.provide(layer)); - - const envelope = yield* machineEnvelope(stdio.stdout[0]!); - expect(envelope.error).toMatchObject({ - // The deduplicated text line appears exactly once, not once per affected service. - message: `The saved stack cannot adopt these changes: [api] port: saved 54321, requested 54999`, - }); - const changes = envelope.stack_changes as ReadonlyArray<{ - readonly service: string; - readonly path: string; - readonly key: string; - }>; - expect(changes).toHaveLength(apiBackedServices.length); - expect(new Set(changes.map((change) => change.service))).toEqual( - new Set(apiBackedServices), - ); - for (const change of changes) - expect(change).toMatchObject({ path: "endpoints.http.port", key: "[api] port" }); - }).pipe(Effect.provide(BunServices.layer)), - ); - - it.live("emits the same structured error fields on the stream-json terminal event", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-stream-json-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "stream-json"\n'); - const fixture = fakeStack(); - yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); - - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "stream-json"\n[db]\nmajor_version = 15\n', - ); - yield* fixture.stack.composition.stop; - const { layer, stdio } = jsonErrorLayers(root, fixture, "stream-json"); - yield* stackStart(flags()).pipe(withJsonErrorHandling, Effect.provide(layer)); - - const event = yield* machineEnvelope(stdio.stdout.at(-1)!); - expect(event.type).toBe("error"); - expect(event.error).toMatchObject({ - code: "ExperimentalStackStartError", - message: expect.stringContaining("[db] major_version: saved 17, requested 15"), - }); - expect(event.stack_changes).toEqual([ - { - service: "database", - path: "config.version", - key: "[db] major_version", - saved: "17", - requested: "15", - editable: true, - }, - ]); - expect(event.recreate_command).toBe(`supabase stack destroy --stack-id ${fixture.stack.id}`); - }).pipe(Effect.provide(BunServices.layer)), - ); - - it.live("always suggests --stack-id for a named stack, naming the stack as plain text", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-named-destroy-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "named-destroy"\n'); - const fixture = fakeStack(); - const target = Layer.succeed(StackTargetResolver, { - resolve: () => - Effect.succeed({ - projectRoot: root, - id: fixture.stack.id, - name: "feature-a", - runtime: "native" as const, - hostRunning: false, - }), - }); - yield* stackStart(flags()).pipe( - Effect.provide(Layer.mergeAll(layers(root, fixture), target)), - ); - - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "named-destroy"\n[db]\nmajor_version = 15\n', - ); - yield* fixture.stack.composition.stop; - const error = yield* stackStart(flags()).pipe( - Effect.provide(Layer.mergeAll(layers(root, fixture), target)), - Effect.flip, - ); - - expect(error).toMatchObject({ - suggestion: expect.stringContaining( - `supabase stack destroy --stack-id ${fixture.stack.id}\` (stack feature-a)`, - ), - }); - expect(error).toBeInstanceOf(StackCommandStartError); - if (error instanceof StackCommandStartError) - expect(error.suggestion).not.toContain("--stack feature-a"); - }).pipe(Effect.provide(BunServices.layer)), - ); - - it.live("names the endpoint a database-only saved stack gains when reopened by the CLI", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-database-only-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "database-only"\n'); - const fixture = fakeStack(); - const excluded = [ - "rest", - "auth", - "realtime", - "storage", - "functions", - "studio", - "mail", - "analytics", - "pooler", - ]; - yield* stackStart(flags(excluded)).pipe(Effect.provide(layers(root, fixture))); - expect(fixture.members.map(({ service }) => service)).toEqual(["database"]); - - const database = fixture.members.find(({ service }) => service === "database"); - if (database?.service !== "database") return yield* Effect.die("Database missing"); - const observed = yield* database.status; - if (observed.config.service !== "database") - return yield* Effect.die("Database config missing"); - // A saved composition created with no public endpoint at all — the CLI's own requests - // always include `endpoints.sql`, but a stack created directly through the package - // (bypassing the CLI) can save a database member without one. - yield* database.restart({ ...observed.config, endpoints: undefined }); - yield* fixture.stack.composition.stop; - - const error = yield* stackStart(flags(excluded)).pipe( - Effect.provide(layers(root, fixture)), - Effect.flip, - ); - - expect(error).toMatchObject({ - reason: "invalid-config", - message: expect.stringContaining( - "database sql endpoint: saved not exposed, requested exposed", - ), - }); - expect(error).toBeInstanceOf(StackCommandStartError); - if (error instanceof StackCommandStartError) { - // This isn't an artifact/version difference, so it must not borrow that wording. - expect(error.suggestion).not.toContain("This CLI release starts a different"); - expect(error.suggestion).toContain( - "This request needs the database sql endpoint that the saved stack doesn't have.", - ); - // The database capability can't be excluded, so destroying is the only offered fix. - expect(error.suggestion).not.toContain("--exclude"); - expect(error.suggestion).toContain(`supabase stack destroy --stack-id ${fixture.stack.id}`); - } - }).pipe(Effect.provide(BunServices.layer)), - ); - - it.live( - "names each endpoint a saved Mail service gains when reopened by the CLI, excluding mail to fix it", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-mail-presence-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "mail-presence"\n'); - const fixture = fakeStack(); - const excluded = [ - "rest", - "auth", - "realtime", - "storage", - "functions", - "studio", - "analytics", - "pooler", - ]; - yield* stackStart(flags(excluded)).pipe(Effect.provide(layers(root, fixture))); - expect(fixture.members.map(({ service }) => service).toSorted()).toEqual([ - "database", - "mail", - ]); - - const mail = fixture.members.find(({ service }) => service === "mail"); - if (mail?.service !== "mail") return yield* Effect.die("Mail missing"); - // A saved Mail service exposing only `smtp` — a composition the package itself can create - // directly (see `Supabase.native.integration.test.ts`), unlike the CLI, which always - // requests all three Mail endpoints. - yield* mail.restart({ service: "mail", config: {}, endpoints: { smtp: { port: "auto" } } }); - yield* fixture.stack.composition.stop; - - const error = yield* stackStart(flags(excluded)).pipe( - Effect.provide(layers(root, fixture)), - Effect.flip, - ); - - expect(error).toMatchObject({ - reason: "invalid-config", - message: expect.stringContaining( - "mail http endpoint: saved not exposed, requested exposed", - ), - }); - expect(error).toMatchObject({ - message: expect.stringContaining( - "mail pop3 endpoint: saved not exposed, requested exposed", - ), - }); - expect(error).toBeInstanceOf(StackCommandStartError); - if (error instanceof StackCommandStartError) { - // A per-endpoint presence change: not an editable port setting, so it must not borrow - // that wording or a config key that implies the user could just set a port. - expect(error.message).not.toContain("saved unset, requested automatic"); - expect(error.message).not.toContain("[local_smtp]"); - expect(error.suggestion).not.toContain("Revert"); - expect(error.suggestion).toContain( - "This request needs the mail http endpoint, mail pop3 endpoint that the saved stack doesn't have.", - ); - // Every incompatible change here belongs to Mail, so excluding it is offered. - expect(error.suggestion).toContain("--exclude mail"); - expect(error.suggestion).toContain( - `supabase stack destroy --stack-id ${fixture.stack.id}`, - ); - } - }).pipe(Effect.provide(BunServices.layer)), - ); - - it.live( - "does not offer --exclude when an unrelated incompatible change would still reject start", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-mail-plus-major-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "mail-plus-major"\n', - ); - const fixture = fakeStack(); - const excluded = [ - "rest", - "auth", - "realtime", - "storage", - "functions", - "studio", - "analytics", - "pooler", - ]; - yield* stackStart(flags(excluded)).pipe(Effect.provide(layers(root, fixture))); - - const mail = fixture.members.find(({ service }) => service === "mail"); - if (mail?.service !== "mail") return yield* Effect.die("Mail missing"); - // Same database-plus-Mail saved stack as above, but this time the config also moves the - // database to a different major version, an incompatibility `--exclude mail` cannot fix. - yield* mail.restart({ service: "mail", config: {}, endpoints: { smtp: { port: "auto" } } }); - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "mail-plus-major"\n[db]\nmajor_version = 15\n', - ); - yield* fixture.stack.composition.stop; - - const error = yield* stackStart(flags(excluded)).pipe( - Effect.provide(layers(root, fixture)), - Effect.flip, - ); - - expect(error).toMatchObject({ - reason: "invalid-config", - message: expect.stringContaining("[db] major_version: saved 17, requested 15"), - }); - expect(error).toMatchObject({ - message: expect.stringContaining( - "mail http endpoint: saved not exposed, requested exposed", - ), - }); - expect(error).toBeInstanceOf(StackCommandStartError); - if (error instanceof StackCommandStartError) { - // `--exclude mail` alone wouldn't fix the database major-version mismatch, so it must - // not be offered as if it were sufficient. - expect(error.suggestion).not.toContain("--exclude"); - expect(error.suggestion).toContain( - `supabase stack destroy --stack-id ${fixture.stack.id}`, - ); - } - }).pipe(Effect.provide(BunServices.layer)), - ); - - it.live( - "pulls Studio into the suggested --exclude set when excluding REST alone would strand it", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-start-exclude-dependency-", - }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "exclude-dependency"\n', - ); - const fixture = fakeStack(); - yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); - - const rest = fixture.members.find(({ service }) => service === "rest"); - if (rest?.service !== "rest") return yield* Effect.die("REST missing"); - const restObserved = yield* rest.status; - if (restObserved.config.service !== "rest") return yield* Effect.die("REST config missing"); - // A saved REST service with its HTTP endpoint omitted — Auth stays exposed and Studio - // stays enabled exactly as the CLI would request them, so a bare `--exclude rest` would - // look sufficient but leaves Studio without the REST API it depends on. - yield* rest.restart({ ...restObserved.config, endpoints: {} }); - yield* fixture.stack.composition.stop; - - const error = yield* stackStart(flags()).pipe( - Effect.provide(layers(root, fixture)), - Effect.flip, - ); - - expect(error).toMatchObject({ - reason: "invalid-config", - message: expect.stringContaining( - "rest http endpoint: saved not exposed, requested exposed", - ), - }); - expect(error).toBeInstanceOf(StackCommandStartError); - if (error instanceof StackCommandStartError) { - // `--exclude rest` alone would itself fail the Studio-without-REST guard, so Studio - // must be folded into the suggested exclusion instead of offered as a broken command. - expect(error.suggestion).toContain("--exclude rest,studio"); - expect(error.suggestion).not.toContain("`--exclude rest`"); - expect(error.suggestion).toContain( - `supabase stack destroy --stack-id ${fixture.stack.id}`, - ); - } - }).pipe(Effect.provide(BunServices.layer)), - ); - - it.live( - "treats an explicit saved artifact version equal to the catalog default as a formality", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-pinned-default-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "pinned-default"\n', - ); - const fixture = fakeStack(); - yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); - const rest = fixture.members.find(({ service }) => service === "rest"); - if (rest?.service !== "rest") return yield* Effect.die("REST missing"); - const restObserved = yield* rest.status; - if (restObserved.config.service !== "rest") return yield* Effect.die("REST config missing"); - const restDefaultVersion = catalogPins().find( - (pin) => pin.service === "rest" && pin.isDefault, - )?.pin.upstreamVersion; - if (restDefaultVersion === undefined) return yield* Effect.die("No default rest pin"); - // Pins the exact version the catalog would pick anyway: a formal difference only, since - // the CLI's own requests never set a top-level `version` and so always fall back to it. - yield* rest.restart({ ...restObserved.config, version: restDefaultVersion }); - yield* fixture.stack.composition.stop; - const error = yield* stackStart(flags()).pipe( - Effect.provide(layers(root, fixture)), - Effect.flip, - ); - - expect(error).toMatchObject({ - reason: "invalid-config", - message: expect.stringContaining( - `rest artifact version: saved ${restDefaultVersion}, requested ${restDefaultVersion}`, - ), - }); - expect(error).toBeInstanceOf(StackCommandStartError); - if (error instanceof StackCommandStartError) { - expect(error.suggestion).not.toContain("This CLI release starts a different"); - expect(error.suggestion).toContain( - "The saved stack pins an explicit rest artifact version that this release would select by default anyway.", - ); - expect(error.suggestion).toContain( - `supabase stack destroy --stack-id ${fixture.stack.id}`, - ); - } - }).pipe(Effect.provide(BunServices.layer)), - ); - it.live("matches a Postgres major alias to the saved pinned database version", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/cli/src/commands/experimental/stack/status/status.handler.ts b/apps/cli/src/commands/experimental/stack/status/status.handler.ts index 8ef1b44d5d..ef57a9683a 100644 --- a/apps/cli/src/commands/experimental/stack/status/status.handler.ts +++ b/apps/cli/src/commands/experimental/stack/status/status.handler.ts @@ -20,8 +20,8 @@ import { } from "../stack-summary.ts"; import { StackApi, - StackTargetError, StackTargetResolver, + mapTargetError, rejectStackOutput, validateStackTarget, } from "../stack.shared.ts"; @@ -82,14 +82,6 @@ type StackReport = { readonly env: Readonly>; }; -const mapTargetError = (error: StackTargetError) => - new StackCommandStatusError({ - reason: error.reason, - message: error.message, - ...(error.suggestion === undefined ? {} : { suggestion: error.suggestion }), - cause: error, - }); - const mapStackError = (error: StackError) => new StackCommandStatusError({ reason: @@ -263,7 +255,7 @@ const findTarget = Effect.fn("experimental.stack.status.findTarget")(function* ( ...(id === undefined ? {} : { id }), runtime: "auto", }) - .pipe(Effect.mapError(mapTargetError)); + .pipe(Effect.mapError(mapTargetError((props) => new StackCommandStatusError(props)))); if (target.id === undefined || target.definition === undefined) return yield* new StackCommandStatusError({ reason: "not-found", @@ -373,7 +365,7 @@ export const stackStatus = Effect.fn("experimental.stack.status")(function* ( yield* validateStackTarget({ stack: Option.getOrUndefined(flags.stack), stackId: Option.getOrUndefined(flags.stackId), - }).pipe(Effect.mapError(mapTargetError)); + }).pipe(Effect.mapError(mapTargetError((props) => new StackCommandStatusError(props)))); if (!flags.env && flags.overrideName.length > 0) return yield* new StackCommandStatusError({ reason: "flags", From 7f736a68b5edbf566cecc4ed059b38585505ffb8 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 15:08:41 +0200 Subject: [PATCH 045/142] fix(stack): re-resolve the backend on proxy retry and carry full unix socket paths HTTP retry reused the backend address captured before the first attempt, risking a connection to a foreign listener that reused the port after the original workload died. Unix ServiceEndpoints held only a directory, forcing every HTTP-readiness consumer to special-case the Postgres socket suffix; the database owner now exports the full socket filename and node-postgres callers derive the directory instead. --- .../stack/src/HttpProxy.integration.test.ts | 65 +++++++++++++++++++ packages/stack/src/HttpProxy.ts | 11 +++- packages/stack/src/Owner.ts | 2 +- packages/stack/src/host/Endpoints.ts | 15 ++--- ...nalyticsVectorImgproxy.integration.test.ts | 9 ++- .../services/AuthStorage.integration.test.ts | 13 ++-- packages/stack/src/services/Catalog.ts | 8 +++ .../services/Functions.integration.test.ts | 15 +++-- .../src/services/Mail.integration.test.ts | 3 +- .../ProcessRecipe.integration.test.ts | 7 +- packages/stack/src/services/ProcessRecipe.ts | 2 +- .../src/services/Realtime.integration.test.ts | 11 ++-- packages/stack/src/services/Recipe.ts | 10 ++- .../src/services/Rest.integration.test.ts | 15 +++-- .../src/services/Vector.integration.test.ts | 3 +- packages/stack/tests/helpers/endpoint.ts | 5 ++ 16 files changed, 146 insertions(+), 48 deletions(-) create mode 100644 packages/stack/tests/helpers/endpoint.ts diff --git a/packages/stack/src/HttpProxy.integration.test.ts b/packages/stack/src/HttpProxy.integration.test.ts index 112ecbdeea..02f4568e45 100644 --- a/packages/stack/src/HttpProxy.integration.test.ts +++ b/packages/stack/src/HttpProxy.integration.test.ts @@ -869,6 +869,71 @@ it.live("gives up after a single retry when the upstream keeps dropping connecti ); }); +it.live( + "re-resolves the backend on retry instead of reaching a listener that reused the invalidated address", + () => { + const logs: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const recovered = createServer((_request, response) => response.end("recovered")); + const recoveredAddress = yield* listen(recovered); + + let foreignConnections = 0; + let resolveInvalidated: (() => void) | undefined; + // One address plays the real backend for the connection that is retried, then the + // address's next owner for any later one, without an OS-level close/rebind race. + let role: "backend" | "foreign" = "backend"; + const shared = createTcpServer((socket) => { + if (role === "backend") { + role = "foreign"; + socket.destroy(); + resolveInvalidated?.(); + } else { + foreignConnections += 1; + socket.destroy(); + } + }); + const sharedAddress = yield* listen(shared); + + const invalidated = yield* Deferred.make(); + yield* Effect.callback((resume) => { + resolveInvalidated = () => resume(Effect.void); + return Effect.void; + }).pipe(Effect.andThen(Deferred.succeed(invalidated, undefined)), Effect.forkScoped); + + let attempts = 0; + const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); + yield* proxy.setRoutes([ + { + id: "retry-target", + prefix: "/", + target: Effect.gen(function* () { + attempts += 1; + if (attempts === 1) return sharedAddress; + yield* Deferred.await(invalidated); + return recoveredAddress; + }), + }, + ]); + + const client = yield* HttpClient.HttpClient; + const response = yield* client.get(`http://127.0.0.1:${proxy.port}/hello`); + expect(response.status).toBe(200); + expect(yield* response.text).toBe("recovered"); + expect(foreignConnections).toBe(0); + expect(logs).toHaveLength(1); + expect(logs[0]).toContain( + "Route retry-target GET upstream failed before responding, retrying", + ); + }), + ).pipe( + Effect.provide( + Layer.mergeAll(NodeHttpClient.layerNodeHttp, NodeServices.layer, captureWarnings(logs)), + ), + ); + }, +); + it.live("closes an upgrade naming the route and cause when a target cannot wake", () => { const logs: Array = []; return Effect.scoped( diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index 0fae17ab2c..e280d72033 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -333,13 +333,20 @@ const forward = Effect.fn("HttpProxy.forward")( const proxyRequest = Effect.fn("HttpProxy.proxyRequest")( (request: IncomingMessage, response: ServerResponse, route: HttpRoute, agent: Agent) => Effect.gen(function* () { - const backend = yield* Effect.raceFirst(route.target, disconnected(request, response)); + // Re-resolved on retry: a backend invalidated between attempts may have its address reused. + const resolveBackend = () => Effect.raceFirst(route.target, disconnected(request, response)); + const backend = yield* resolveBackend(); yield* forward(request, response, route, backend, isReplayable(request) ? agent : false).pipe( Effect.catchIf(isRetryable(request, response), (error) => Effect.logWarning( `Route ${route.id} ${request.method ?? "GET"} upstream failed before responding, retrying`, error, - ).pipe(Effect.andThen(forward(request, response, route, backend, false))), + ).pipe( + Effect.andThen(resolveBackend()), + Effect.andThen((retryBackend) => + forward(request, response, route, retryBackend, false), + ), + ), ), ); }), diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index 665c401c20..d587ba3ee9 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -560,7 +560,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { .acquire(id, name !== "inspector", `traffic on endpoint ${name}`) .pipe( Effect.andThen(recipe.endpoint(name)), - Effect.flatMap(backendAddress), + Effect.map(backendAddress), Effect.mapError((cause) => cause instanceof ProxyError ? cause diff --git a/packages/stack/src/host/Endpoints.ts b/packages/stack/src/host/Endpoints.ts index 036f245874..09a0b32222 100644 --- a/packages/stack/src/host/Endpoints.ts +++ b/packages/stack/src/host/Endpoints.ts @@ -2,7 +2,7 @@ import { allowedEndpointNames, type ServiceCreation } from "../services/Catalog. import type { ServiceEndpoint } from "../services/Recipe.ts"; import { apiPath as storageApiPath } from "../services/Storage.ts"; import type { NetworkEndpoint } from "../Network.ts"; -import { ProxyError, type BackendAddress } from "../Proxy.ts"; +import type { BackendAddress } from "../Proxy.ts"; import { Data, Effect, Redacted } from "effect"; export class EndpointError extends Data.TaggedError("EndpointError")<{ @@ -96,15 +96,10 @@ export const joinRoutes = (creation: ServiceCreation, name: string): NetworkEndp : undefined; /** Translates a launched runtime's endpoint into the proxy's backend address. */ -export const backendAddress = ( - endpoint: ServiceEndpoint, -): Effect.Effect => { - if (endpoint.kind === "unix") - return endpoint.path === undefined - ? Effect.fail(new ProxyError({ message: "Unix endpoint has no path" })) - : Effect.succeed({ path: `${endpoint.path}/.s.PGSQL.${endpoint.port}` }); - return Effect.succeed({ host: endpoint.host ?? "127.0.0.1", port: endpoint.port }); -}; +export const backendAddress = (endpoint: ServiceEndpoint): BackendAddress => + endpoint.kind === "unix" + ? { path: endpoint.path } + : { host: endpoint.host ?? "127.0.0.1", port: endpoint.port }; export const publicUrl = (host: string, port: number) => `http://${host}:${port}`; diff --git a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts index cae39c79f0..6c74eca221 100644 --- a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts +++ b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts @@ -6,6 +6,7 @@ import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; +import { httpHost } from "../../tests/helpers/endpoint.ts"; import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; @@ -79,7 +80,7 @@ describe("service catalog", () => { const analyticsRelay = yield* makeDockerHttpRelay(analyticsRecipe.endpoint("http")); const analyticsResponse = yield* client.execute( HttpClientRequest.get( - `http://${analyticsEndpoint.host}:${analyticsEndpoint.port}/health`, + `http://${httpHost(analyticsEndpoint)}:${analyticsEndpoint.port}/health`, ), ); expect(analyticsResponse.status).toBe(200); @@ -108,7 +109,9 @@ describe("service catalog", () => { yield* vector.ready; const vectorEndpoint = yield* vectorRecipe.endpoint("http"); const vectorResponse = yield* client.execute( - HttpClientRequest.get(`http://${vectorEndpoint.host}:${vectorEndpoint.port}/health`), + HttpClientRequest.get( + `http://${httpHost(vectorEndpoint)}:${vectorEndpoint.port}/health`, + ), ); expect(vectorResponse.status).toBe(200); @@ -127,7 +130,7 @@ describe("service catalog", () => { const imgproxyEndpoint = yield* imgproxyRecipe.endpoint("http"); const imgproxyResponse = yield* client.execute( HttpClientRequest.get( - `http://${imgproxyEndpoint.host}:${imgproxyEndpoint.port}/health`, + `http://${httpHost(imgproxyEndpoint)}:${imgproxyEndpoint.port}/health`, ), ); expect(imgproxyResponse.status).toBe(200); diff --git a/packages/stack/src/services/AuthStorage.integration.test.ts b/packages/stack/src/services/AuthStorage.integration.test.ts index dc6118829a..b06f83a5d2 100644 --- a/packages/stack/src/services/AuthStorage.integration.test.ts +++ b/packages/stack/src/services/AuthStorage.integration.test.ts @@ -7,6 +7,7 @@ import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; +import { httpHost } from "../../tests/helpers/endpoint.ts"; import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; @@ -78,7 +79,7 @@ describe("service catalog", () => { const email = "catalog@example.test"; const password = "catalog-password-123"; const signupRequest = yield* HttpClientRequest.bodyJson({ email, password })( - HttpClientRequest.post(`http://${authEndpoint.host}:${authEndpoint.port}/signup`), + HttpClientRequest.post(`http://${httpHost(authEndpoint)}:${authEndpoint.port}/signup`), ); const signup = yield* client.execute(signupRequest); expect(signup.status).toBe(200); @@ -92,7 +93,7 @@ describe("service catalog", () => { password, })( HttpClientRequest.post( - `http://${authEndpoint.host}:${authEndpoint.port}/token?grant_type=password`, + `http://${httpHost(authEndpoint)}:${authEndpoint.port}/token?grant_type=password`, ), ); const login = yield* client.execute(loginRequest); @@ -145,7 +146,7 @@ describe("service catalog", () => { ); const bucketRequest = yield* HttpClientRequest.bodyJson({ name: "catalog" })( HttpClientRequest.post( - `http://${storageEndpoint.host}:${storageEndpoint.port}/bucket`, + `http://${httpHost(storageEndpoint)}:${storageEndpoint.port}/bucket`, ).pipe( HttpClientRequest.setHeader("Authorization", `Bearer ${serviceToken}`), HttpClientRequest.setHeader("apikey", serviceToken), @@ -155,7 +156,7 @@ describe("service catalog", () => { expect(bucket.status).toBe(200); const uploadRequest = HttpClientRequest.bodyText( HttpClientRequest.post( - `http://${storageEndpoint.host}:${storageEndpoint.port}/object/catalog/hello.txt`, + `http://${httpHost(storageEndpoint)}:${storageEndpoint.port}/object/catalog/hello.txt`, ).pipe( HttpClientRequest.setHeader("Authorization", `Bearer ${serviceToken}`), HttpClientRequest.setHeader("apikey", serviceToken), @@ -173,7 +174,7 @@ describe("service catalog", () => { ); const imageRequest = HttpClientRequest.bodyUint8Array( HttpClientRequest.post( - `http://${storageEndpoint.host}:${storageEndpoint.port}/object/catalog/source.png`, + `http://${httpHost(storageEndpoint)}:${storageEndpoint.port}/object/catalog/source.png`, ).pipe( HttpClientRequest.setHeader("Authorization", `Bearer ${serviceToken}`), HttpClientRequest.setHeader("apikey", serviceToken), @@ -185,7 +186,7 @@ describe("service catalog", () => { expect(imageUpload.status).toBe(200); const transformedImage = yield* client.execute( HttpClientRequest.get( - `http://${storageEndpoint.host}:${storageEndpoint.port}/render/image/authenticated/catalog/source.png?width=1&height=1`, + `http://${httpHost(storageEndpoint)}:${storageEndpoint.port}/render/image/authenticated/catalog/source.png?width=1&height=1`, ).pipe( HttpClientRequest.setHeader("Authorization", `Bearer ${serviceToken}`), HttpClientRequest.setHeader("apikey", serviceToken), diff --git a/packages/stack/src/services/Catalog.ts b/packages/stack/src/services/Catalog.ts index 05779b8fe7..bbd735bd9e 100644 --- a/packages/stack/src/services/Catalog.ts +++ b/packages/stack/src/services/Catalog.ts @@ -28,6 +28,7 @@ import { type CatalogOptions, type CatalogRecipe as RecipeCatalogRecipe, type ProcessRecipeResult, + type ServiceEndpoint, } from "./Recipe.ts"; import { makeProcessRecipe, type ProcessDependencies } from "./ProcessRecipe.ts"; import { borrow } from "../namespace/Paths.ts"; @@ -273,6 +274,13 @@ const databaseRecipe = ( endpoint: (name) => name === "sql" ? component.endpoint.pipe( + // node-postgres appends the `.s.PGSQL.` suffix itself, so the database keeps the + // directory and port private; the catalog's endpoint carries the full socket filename. + Effect.map((value): ServiceEndpoint => + value.kind === "unix" + ? { ...value, path: `${value.path}/.s.PGSQL.${value.port}` } + : value, + ), Effect.mapError( (cause) => new CatalogError({ diff --git a/packages/stack/src/services/Functions.integration.test.ts b/packages/stack/src/services/Functions.integration.test.ts index aed0557564..7cfed8c6a7 100644 --- a/packages/stack/src/services/Functions.integration.test.ts +++ b/packages/stack/src/services/Functions.integration.test.ts @@ -24,6 +24,7 @@ import * as Functions from "./Functions.ts"; import { makeProcessRecipe } from "./ProcessRecipe.ts"; import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; +import { httpHost } from "../../tests/helpers/endpoint.ts"; const options = (root: string) => ({ stackId: "catalog-functions", @@ -117,7 +118,7 @@ describe("service catalog", () => { yield* instance.ready; const endpoint = yield* recipe.endpoint("http"); const response = yield* client.execute( - HttpClientRequest.get("http://" + endpoint.host + ":" + endpoint.port + "/hello"), + HttpClientRequest.get("http://" + httpHost(endpoint) + ":" + endpoint.port + "/hello"), ); expect(response.status).toBe(200); expect(yield* response.json).toEqual( @@ -136,7 +137,7 @@ describe("service catalog", () => { const inspector = yield* recipe.endpoint("inspector"); const inspectorResponse = yield* client.execute( HttpClientRequest.get( - "http://" + inspector.host + ":" + inspector.port + "/json/version", + "http://" + httpHost(inspector) + ":" + inspector.port + "/json/version", ), ); expect(inspectorResponse.status).toBe(200); @@ -285,7 +286,9 @@ describe("service catalog", () => { Effect.tapError(() => Ref.get(logs).pipe(Effect.flatMap(Effect.logError))), ); const endpoint = yield* recipe.endpoint("http"); - const response = yield* client.get(`http://${endpoint.host}:${endpoint.port}/hello`); + const response = yield* client.get( + `http://${httpHost(endpoint)}:${endpoint.port}/hello`, + ); expect(response.status, yield* Ref.get(logs)).toBe(200); expect(yield* response.text).toBe("shared"); yield* instance.stop; @@ -359,7 +362,7 @@ describe("service catalog", () => { ); const endpoint = yield* recipe.endpoint("http"); - const response = yield* client.get(`http://${endpoint.host}:${endpoint.port}/hello`); + const response = yield* client.get(`http://${httpHost(endpoint)}:${endpoint.port}/hello`); expect(response.status, yield* Ref.get(logs)).toBe(200); expect(yield* response.text).toBe("config"); @@ -442,7 +445,7 @@ describe("service catalog", () => { ); const endpoint = yield* recipe.endpoint("http"); - const response = yield* client.get(`http://${endpoint.host}:${endpoint.port}/hello`); + const response = yield* client.get(`http://${httpHost(endpoint)}:${endpoint.port}/hello`); yield* Deferred.await(warned).pipe( Effect.timeout("30 seconds"), Effect.tapError(() => Ref.get(logs).pipe(Effect.flatMap(Effect.logError))), @@ -558,7 +561,7 @@ for (const runtime of ["native", "docker"] as const) { ), ); const endpoint = yield* recipe.endpoint("http"); - const base = `http://${endpoint.host}:${endpoint.port}`; + const base = `http://${httpHost(endpoint)}:${endpoint.port}`; const response = yield* client.execute(HttpClientRequest.get(`${base}/hello`)); const responseText = yield* response.text; expect(response.status, responseText).toBe(200); diff --git a/packages/stack/src/services/Mail.integration.test.ts b/packages/stack/src/services/Mail.integration.test.ts index 5142dd05ab..e64820af7b 100644 --- a/packages/stack/src/services/Mail.integration.test.ts +++ b/packages/stack/src/services/Mail.integration.test.ts @@ -6,6 +6,7 @@ import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; +import { httpHost } from "../../tests/helpers/endpoint.ts"; const options = (root: string) => ({ stackId: "catalog-mail", @@ -78,7 +79,7 @@ describe("service catalog", () => { yield* instance.ready; const endpoint = yield* recipe.endpoint("http"); const response = yield* client.execute( - HttpClientRequest.get(`http://${endpoint.host}:${endpoint.port}/readyz`), + HttpClientRequest.get(`http://${httpHost(endpoint)}:${endpoint.port}/readyz`), ); expect(response.status).toBe(200); yield* instance.stop; diff --git a/packages/stack/src/services/ProcessRecipe.integration.test.ts b/packages/stack/src/services/ProcessRecipe.integration.test.ts index 5b4974efb4..1d4b54bdab 100644 --- a/packages/stack/src/services/ProcessRecipe.integration.test.ts +++ b/packages/stack/src/services/ProcessRecipe.integration.test.ts @@ -40,6 +40,7 @@ import { } from "../runtime/Container.ts"; import { ServiceError } from "../Service.ts"; import { makeStandaloneService } from "../../tests/standalone-service.ts"; +import { httpHost } from "../../tests/helpers/endpoint.ts"; import { makeProcessRecipe, type ProcessDependencies, @@ -773,7 +774,7 @@ describe("process recipe startup", () => { expect(yield* Ref.get(taken)).toHaveLength(2); expect(yield* Ref.get(taken)).not.toContain(endpoint?.port); const response = yield* client.execute( - HttpClientRequest.get(`http://${endpoint?.host}:${endpoint?.port}/`), + HttpClientRequest.get(`http://${endpoint && httpHost(endpoint)}:${endpoint?.port}/`), ); expect(yield* response.text).toBe("owned-fixture"); yield* service.stop; @@ -1000,7 +1001,9 @@ describe("process recipe startup", () => { ); expect(yield* competingResponse.text).toBe("competing-listener"); const response = yield* client.execute( - HttpClientRequest.get(`http://${endpoint?.host}:${endpoint?.port}/api/health`), + HttpClientRequest.get( + `http://${endpoint && httpHost(endpoint)}:${endpoint?.port}/api/health`, + ), ); expect(yield* response.text).toBe(`owned-fixture:${endpoint?.port}`); expect(yield* Ref.get(collisionInstalled)).toBe(true); diff --git a/packages/stack/src/services/ProcessRecipe.ts b/packages/stack/src/services/ProcessRecipe.ts index c75eccfc31..7a5f02728c 100644 --- a/packages/stack/src/services/ProcessRecipe.ts +++ b/packages/stack/src/services/ProcessRecipe.ts @@ -382,7 +382,7 @@ const readiness = Effect.fn("ProcessRecipe.readiness")(function* ( timeout: Duration.Input = "60 seconds", ) { const attempt = client - .execute(HttpClientRequest.get(`http://${endpoint.host}:${endpoint.port}${path}`)) + .execute(HttpClientRequest.get(`http://127.0.0.1:${endpoint.port}${path}`)) .pipe( Effect.flatMap((response) => (response.status >= 200 && response.status < 300) || response.status === 401 diff --git a/packages/stack/src/services/Realtime.integration.test.ts b/packages/stack/src/services/Realtime.integration.test.ts index deaa617bdc..d69ee032ed 100644 --- a/packages/stack/src/services/Realtime.integration.test.ts +++ b/packages/stack/src/services/Realtime.integration.test.ts @@ -8,6 +8,7 @@ import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-rela import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; +import { httpHost } from "../../tests/helpers/endpoint.ts"; const options = (root: string) => ({ stackId: "catalog-realtime", @@ -68,7 +69,7 @@ describe("service catalog", () => { const realtimeEndpoint = yield* realtimeRecipe.endpoint("http"); const realtimeResponse = yield* client.execute( HttpClientRequest.get( - `http://${realtimeEndpoint.host}:${realtimeEndpoint.port}/healthcheck`, + `http://${httpHost(realtimeEndpoint)}:${realtimeEndpoint.port}/healthcheck`, ), ); expect(realtimeResponse.status).toBe(200); @@ -86,7 +87,9 @@ describe("service catalog", () => { const pgmetaEndpoint = yield* pgmetaRecipe.endpoint("http"); const pgmetaRelay = yield* makeDockerHttpRelay(pgmetaRecipe.endpoint("http")); const schemasResponse = yield* client.execute( - HttpClientRequest.get(`http://${pgmetaEndpoint.host}:${pgmetaEndpoint.port}/schemas`), + HttpClientRequest.get( + `http://${httpHost(pgmetaEndpoint)}:${pgmetaEndpoint.port}/schemas`, + ), ); expect(schemasResponse.status).toBe(200); expect(yield* schemasResponse.text).toContain("public"); @@ -114,13 +117,13 @@ describe("service catalog", () => { const studioEndpoint = yield* studioRecipe.endpoint("http"); const profileResponse = yield* client.execute( HttpClientRequest.get( - `http://${studioEndpoint.host}:${studioEndpoint.port}/api/platform/profile`, + `http://${httpHost(studioEndpoint)}:${studioEndpoint.port}/api/platform/profile`, ), ); expect(profileResponse.status).toBe(200); const queryResponse = yield* client.execute( HttpClientRequest.post( - `http://${studioEndpoint.host}:${studioEndpoint.port}/api/platform/pg-meta/default/query`, + `http://${httpHost(studioEndpoint)}:${studioEndpoint.port}/api/platform/pg-meta/default/query`, ).pipe(HttpClientRequest.bodyJsonUnsafe({ query: "select current_user" })), ); expect(queryResponse.status).toBe(200); diff --git a/packages/stack/src/services/Recipe.ts b/packages/stack/src/services/Recipe.ts index 986bf1011b..38bbe8de45 100644 --- a/packages/stack/src/services/Recipe.ts +++ b/packages/stack/src/services/Recipe.ts @@ -17,12 +17,10 @@ export class CatalogError extends Data.TaggedError("CatalogError")<{ readonly cause?: unknown; }> {} -export interface ServiceEndpoint { - readonly kind: "tcp" | "unix"; - readonly host?: "127.0.0.1"; - readonly path?: string; - readonly port: number; -} +export type ServiceEndpoint = + | { readonly kind: "tcp"; readonly host?: "127.0.0.1"; readonly port: number } + /** `path` is the full socket filename, for example `/.s.PGSQL.`. */ + | { readonly kind: "unix"; readonly path: string; readonly port: number }; export const EndpointIntent = Schema.Struct({ port: Schema.Union([Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)), Schema.Literal("auto")]), diff --git a/packages/stack/src/services/Rest.integration.test.ts b/packages/stack/src/services/Rest.integration.test.ts index 5d3ad83bda..7658aa96b6 100644 --- a/packages/stack/src/services/Rest.integration.test.ts +++ b/packages/stack/src/services/Rest.integration.test.ts @@ -13,6 +13,7 @@ import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; +import { httpHost } from "../../tests/helpers/endpoint.ts"; const makeTestState = (root: string) => Layer.build(StackNamespace.layer({ root })).pipe( @@ -159,7 +160,7 @@ describe("service catalog", () => { enabled: Ref.get(restActive), backend: restRecipe.endpoint("http").pipe( Effect.flatMap((endpoint) => - endpoint.host === undefined + endpoint.kind !== "tcp" || endpoint.host === undefined ? Effect.fail(new ProxyError({ message: "REST endpoint has no host" })) : Effect.succeed({ host: endpoint.host, port: endpoint.port }), ), @@ -206,6 +207,7 @@ describe("service catalog", () => { Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const client = yield* HttpClient.HttpClient; const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-rest-native-" }); const stackId = "catalog-native"; @@ -229,11 +231,14 @@ describe("service catalog", () => { yield* database.start; yield* database.ready; const databaseEndpoint = yield* databaseRecipe.endpoint("sql"); - if (databaseEndpoint.kind !== "unix" || databaseEndpoint.path === undefined) + if (databaseEndpoint.kind !== "unix") return yield* new ProxyError({ message: "Native database did not expose a Unix socket" }); + // node-postgres appends the `.s.PGSQL.` suffix itself, so it must receive the + // socket directory rather than the endpoint's full socket filename. + const databaseSocketDir = path.dirname(databaseEndpoint.path); const databaseLayer = yield* Layer.build( PgClient.layer({ - host: databaseEndpoint.path, + host: databaseSocketDir, port: databaseEndpoint.port, database: "postgres", username: "supabase_admin", @@ -255,7 +260,7 @@ describe("service catalog", () => { config: { databaseUrl: "postgresql://supabase_admin:postgres@127.0.0.1/postgres?host=" + - encodeURIComponent(databaseEndpoint.path), + encodeURIComponent(databaseSocketDir), jwtSecret: secret, anonRole: "anon", }, @@ -278,7 +283,7 @@ describe("service catalog", () => { ); const response = yield* client.execute( HttpClientRequest.get( - "http://" + endpoint.host + ":" + endpoint.port + "/catalog_native_probe", + "http://" + httpHost(endpoint) + ":" + endpoint.port + "/catalog_native_probe", ).pipe(HttpClientRequest.setHeader("Authorization", "Bearer " + token)), ); const responseBody = yield* response.text; diff --git a/packages/stack/src/services/Vector.integration.test.ts b/packages/stack/src/services/Vector.integration.test.ts index 2b752c3bd6..1b022030a0 100644 --- a/packages/stack/src/services/Vector.integration.test.ts +++ b/packages/stack/src/services/Vector.integration.test.ts @@ -7,6 +7,7 @@ import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; +import { httpHost } from "../../tests/helpers/endpoint.ts"; const options = (root: string, runtime: "docker" | "native") => ({ stackId: "catalog-test", @@ -47,7 +48,7 @@ describe("vector recipe", () => { yield* vector.ready; const endpoint = yield* recipe.endpoint("http"); const response = yield* client.execute( - HttpClientRequest.get(`http://${endpoint.host}:${endpoint.port}/health`), + HttpClientRequest.get(`http://${httpHost(endpoint)}:${endpoint.port}/health`), ); expect(response.status).toBe(200); yield* fs.makeDirectory(`${root}/vector/runtime/vector/.vector-api.yaml-interrupted`); diff --git a/packages/stack/tests/helpers/endpoint.ts b/packages/stack/tests/helpers/endpoint.ts new file mode 100644 index 0000000000..54fb86feec --- /dev/null +++ b/packages/stack/tests/helpers/endpoint.ts @@ -0,0 +1,5 @@ +import type { ServiceEndpoint } from "../../src/services/Recipe.ts"; + +/** The host to dial for an HTTP-readiness endpoint, which is always TCP in practice. */ +export const httpHost = (endpoint: ServiceEndpoint): string => + endpoint.kind === "unix" ? endpoint.path : (endpoint.host ?? "127.0.0.1"); From 099532dc1ce39356121b5653b2a6a9ec80353758 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 16:31:33 +0200 Subject: [PATCH 046/142] fix(stack): keep a destroyed stack's directories so a same-id restart reuses them --- packages/stack/src/Sweep.integration.test.ts | 2 +- packages/stack/src/lifetime.integration.test.ts | 6 +++--- .../stack/src/namespace/Namespace.integration.test.ts | 5 +++-- packages/stack/src/namespace/Registry.ts | 8 ++------ packages/stack/src/testing.integration.test.ts | 4 +++- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/packages/stack/src/Sweep.integration.test.ts b/packages/stack/src/Sweep.integration.test.ts index 2dbadb7efa..3e58be9bf7 100644 --- a/packages/stack/src/Sweep.integration.test.ts +++ b/packages/stack/src/Sweep.integration.test.ts @@ -201,7 +201,7 @@ it.live.skipIf(process.platform === "win32")( const swept = yield* Effect.all( [ awaitDestroyed(orphan, since), - awaitRemoval(rootA, sessionId), + awaitRemoval(path.join(rootA, sessionId), "state.json"), awaitRemoval(path.join(rootA, deadId), "owner.json"), ], { concurrency: "unbounded" }, diff --git a/packages/stack/src/lifetime.integration.test.ts b/packages/stack/src/lifetime.integration.test.ts index d68a26d701..b2b692bc24 100644 --- a/packages/stack/src/lifetime.integration.test.ts +++ b/packages/stack/src/lifetime.integration.test.ts @@ -129,7 +129,7 @@ it.live.skipIf(process.platform === "win32")( const owned = yield* descendantsOf(ownerPid); expect(owned.length, "the owner runs the native mail service").toBeGreaterThan(0); - const removed = yield* watchEntry(stateRoot, stackId, false); + const removed = yield* watchEntry(`${stateRoot}/${stackId}`, "state.json", false); const released = yield* watchLeaseRelease(stateRoot, stackId); yield* creator.kill({ killSignal: "SIGKILL" }); yield* removed.pipe( @@ -143,7 +143,7 @@ it.live.skipIf(process.platform === "win32")( // Releasing the lease is the owner's last act, after its native processes have exited. yield* released; expect(owned.filter(alive), "native processes die with their owner").toEqual([]); - expect(yield* fs.exists(`${stateRoot}/${stackId}`)).toBe(false); + expect(yield* fs.exists(`${stateRoot}/${stackId}/state.json`)).toBe(false); expect(yield* state.read(stackId)).toBeUndefined(); }).pipe(Effect.scoped, Effect.provide(layer)), { timeout: 180_000 }, @@ -169,7 +169,7 @@ it.live("destroys a session stack when its creating handle closes", () => yield* assertOwnerExited(ownerPid); const state = yield* stateFor(locations.stateRoot); expect(yield* state.read(id)).toBeUndefined(); - expect(yield* fs.exists(`${locations.stateRoot}/${id}`)).toBe(false); + expect(yield* fs.exists(`${locations.stateRoot}/${id}/state.json`)).toBe(false); }).pipe(Effect.scoped, Effect.provide(layer)), ); diff --git a/packages/stack/src/namespace/Namespace.integration.test.ts b/packages/stack/src/namespace/Namespace.integration.test.ts index 69e5017713..2610200f58 100644 --- a/packages/stack/src/namespace/Namespace.integration.test.ts +++ b/packages/stack/src/namespace/Namespace.integration.test.ts @@ -321,7 +321,7 @@ describe("durable registry", () => { ), ); - it.live("removes empty stack parents without deleting unowned data", () => + it.live("unregisters a stack and empties its data without deleting unowned data", () => run( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -331,7 +331,8 @@ describe("durable registry", () => { yield* store.save(initial); yield* fs.makeDirectory(path.join(root, initial.id, "data"), { recursive: true }); yield* store.remove(initial.id); - expect(yield* fs.exists(path.join(root, initial.id))).toBe(false); + expect(yield* fs.exists(path.join(root, initial.id, "state.json"))).toBe(false); + expect(yield* fs.readDirectory(path.join(root, initial.id, "data"))).toEqual([]); yield* store.save(initial); yield* fs.makeDirectory(path.join(root, initial.id, "data"), { recursive: true }); diff --git a/packages/stack/src/namespace/Registry.ts b/packages/stack/src/namespace/Registry.ts index 1698b6afef..365b0c9a71 100644 --- a/packages/stack/src/namespace/Registry.ts +++ b/packages/stack/src/namespace/Registry.ts @@ -5,7 +5,6 @@ import { namespaceError, retryTransientRead, type NamespaceError } from "./Capab import { CLAIMS_FILE } from "./Claims.ts"; import { OWNER_FILE, OWNER_LOG_FILE } from "./Lease.ts"; import * as Publication from "./Publication.ts"; -import { removeEmptyDirectory } from "./drivers/FileSystem.ts"; import { acquireLock, isBusy, takeLock } from "./drivers/Sqlite.ts"; const SafeId = Schema.String.pipe( @@ -194,11 +193,8 @@ export const make = ( yield* fs .remove(file, { force: true }) .pipe(Effect.mapError((cause) => namespaceError("remove", cause))); - // The container-env scratch directory is removed registration-independently by the owner - // (Owner.ts's `removeContainerEnvRoot`, shared by destroy and abandonment); by the time this - // runs for a confirmed destroy, `data` is already empty of it. - yield* removeEmptyDirectory(path.join(stackRoot(id), "data")); - yield* removeEmptyDirectory(stackRoot(id)); + // The stack and data directories stay: deleting and recreating them for a restart under the + // same id lets the container engine's file share report fresh bind sources as missing. }); /** * One attempt: forks `attemptScope` under `guardScope` before opening anything, so there is diff --git a/packages/stack/src/testing.integration.test.ts b/packages/stack/src/testing.integration.test.ts index f35d4e1311..4e139594e1 100644 --- a/packages/stack/src/testing.integration.test.ts +++ b/packages/stack/src/testing.integration.test.ts @@ -211,7 +211,9 @@ it.live( expect(Exit.isSuccess(yield* Fiber.await(snapshot))).toBe(true); expect(yield* discover({ stateRoot })).toEqual([]); - expect(yield* fs.exists(path.join(stateRoot, test.stack.id))).toBe(false); + expect(yield* fs.exists(path.join(stateRoot, test.stack.id, "state.json"))).toBe(false); + const data = path.join(stateRoot, test.stack.id, "data"); + expect((yield* fs.exists(data)) ? yield* fs.readDirectory(data) : []).toEqual([]); }).pipe( Effect.scoped, Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp)), From cb59cb4e8672cf6f3b9139b72ff4f642ac747c53 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 16:31:33 +0200 Subject: [PATCH 047/142] refactor(stack): share one shutdown body between stop and destroy --- packages/stack/src/StackHost.ts | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index 34354edcde..aa445cf167 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -86,9 +86,8 @@ const hostError = (operation: string, cause: unknown, reason?: "runtime-unavaila /** * Matches an engine CLI that is missing or reports a daemon that is not listening, never one - * that rejects the caller (for example on permissions): the one startup-diagnostic distinction - * `StackHostError`'s own `reason` has carried since before phase 2d, so `destroy`/`shutdown` - * (`effect.ts`) know to leave a stack registered for retry instead of surfacing a hard failure. + * that rejects the caller (for example on permissions), so `destroy`/`shutdown` (`effect.ts`) + * know to leave a stack registered for retry instead of surfacing a hard failure. * Unrelated to abandonment's own cleanup-retry decision, which probes the engine directly instead * of matching a message at all. */ @@ -282,11 +281,6 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( yield* Deferred.succeed(exit, undefined); } }); - const stopOwned = Effect.gen(function* () { - yield* attachments.stopAll; - yield* runner.cleanup; - yield* owner.namespace.stop; - }); const body: Effect.Effect = mode === "abandon" ? Effect.gen(function* () { @@ -310,19 +304,14 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( }); yield* finish; }) - : mode === "stop" - ? stopOwned.pipe( - Effect.andThen(finish), - Effect.mapError((cause) => stackError("shutdown", cause)), - ) - : Effect.gen(function* () { - yield* attachments.stopAll; - yield* runner.cleanup; - yield* owner.namespace.destroy; - }).pipe( - Effect.andThen(finish), - Effect.mapError((cause) => stackError("shutdown", cause)), - ); + : Effect.gen(function* () { + yield* attachments.stopAll; + yield* runner.cleanup; + yield* mode === "stop" ? owner.namespace.stop : owner.namespace.destroy; + }).pipe( + Effect.andThen(finish), + Effect.mapError((cause) => stackError("shutdown", cause)), + ); const cleanup = body.pipe( Effect.catchCause((cause) => mode !== "abandon" From 5ffd013b09114fec9553e142dae0f178c504068a Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 16:40:36 +0200 Subject: [PATCH 048/142] refactor(stack): surface a failed destroy without waiting for an owner that keeps running --- packages/stack/src/effect.integration.test.ts | 4 ++-- packages/stack/src/effect.ts | 24 ++----------------- 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/packages/stack/src/effect.integration.test.ts b/packages/stack/src/effect.integration.test.ts index 6991e864b5..c8be0f1e4d 100644 --- a/packages/stack/src/effect.integration.test.ts +++ b/packages/stack/src/effect.integration.test.ts @@ -147,7 +147,7 @@ it.live("starts the owner on opt-in reopen without starting saved services", () ); it.live( - "stops the owner after destroy fails and retains saved data for retry", + "reports a failed destroy and keeps the owner and saved data for retry", () => Effect.scoped( Effect.gen(function* () { @@ -199,7 +199,7 @@ it.live( expect(yield* fs.readLink(dataRoot).pipe(Effect.isSuccess)).toBe(true); const retained = yield* discover(options); expect(retained).toHaveLength(1); - expect(retained[0]?.host).toBeUndefined(); + expect(retained[0]?.host?.pid).toBe(running[0]?.host?.pid); yield* fs.remove(dataRoot, { force: true }); yield* stack.destroy; diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index a82ba39fe5..bfe977e438 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -528,31 +528,11 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( Effect.provideContext(services), ); if (endpoint === undefined) return { runtimeCleanup: "complete" } as const; - if (Exit.isFailure(refusal)) { - const shutdownFailure = Option.match(Cause.findErrorOption(refusal.cause), { + if (Exit.isFailure(refusal)) + return yield* Option.match(Cause.findErrorOption(refusal.cause), { onNone: () => failure(operation, Cause.pretty(refusal.cause)), onSome: (cause) => failure(operation, cause), }); - if (!destroy) return yield* shutdownFailure; - const exitResult = yield* waitForOwnerExit(endpoint.pid, ownerExitProbe(fs)).pipe( - Effect.mapError((cause) => failure("shutdown-exit", cause)), - Effect.exit, - ); - if (Exit.isFailure(exitResult)) { - const exitFailure = Option.match(Cause.findErrorOption(exitResult.cause), { - onNone: () => failure("shutdown-exit", Cause.pretty(exitResult.cause)), - onSome: (cause) => failure("shutdown-exit", cause), - }); - const exitStatus = exitFailure.message.includes("still running") - ? "Owner is still running after failed destroy" - : "Owner exit was not confirmed after failed destroy"; - return yield* new StackError({ - ...shutdownFailure, - message: `${shutdownFailure.message}; ${exitStatus}; exit probe: ${exitFailure.message}`, - }); - } - return yield* shutdownFailure; - } yield* waitForOwnerExit(endpoint.pid, ownerExitProbe(fs)).pipe( Effect.mapError((cause) => failure("shutdown-exit", cause)), ); From 062b354173111a2b69f12bca433c3718416c1f31 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 17:03:28 +0200 Subject: [PATCH 049/142] refactor(stack): drop the forwarding instance cleanup wrapper --- packages/stack/src/Owner.ts | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index d587ba3ee9..4b39e58e11 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -208,14 +208,6 @@ interface Entry extends Orchestrator.RegisteredInstance { /** The saved creation, which composition wiring and launches update. */ readonly creation: Ref.Ref; readonly namespace: NetworkNamespace; - /** - * Removes this instance's containers, storage namespace and network namespace, independent of - * the saved registration: used by destroy (which then publishes the registration removal) and - * reusable as-is by a future abandonment path, which releases without touching the registration. - */ - readonly cleanupResources: ( - context: ServiceInstanceContext, - ) => Effect.Effect; } const isRecord = (value: unknown): value is Readonly> => @@ -513,14 +505,6 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { }); const initial = recipe.creation; const creation = yield* Ref.make(initial); - // Independent of the saved registration and of on-disk discovery: removes containers and the - // owned storage namespace. Shared as-is by destroy and abandonment, through `core.removeData`; - // neither reads or writes the registration. Network reservations are never part of this: they - // are destroy's own separate `release` step (`Orchestrator.destroy`), never abandonment's, - // since a stale reservation stays for `Ports.ts`'s lazy reclamation once the holder's - // registration is confirmed gone, the abandonment trigger. - const cleanupResources = (context: ServiceInstanceContext) => - recipe.definition.removeData(context); const core = yield* makeService( { ...recipe.definition, @@ -529,10 +513,6 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { Effect.mapError(serviceError("state")), Effect.andThen(recipe.definition.launch(context)), ), - // Registration-free: `core.removeData` runs this inside its execution lock and, only when - // a `confirm` step is given, publishes the registration removal afterward (destroy passes - // it; abandonment never does). This is the one path both destroy and abandonment share. - removeData: cleanupResources, }, { id, config: initial, report: orchestrator.report }, ).pipe(Effect.provideService(Scope.Scope, ownerScope)); @@ -582,7 +562,6 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { recipe, creation, namespace, - cleanupResources, confirmRemoved: removeInstanceRegistration(id).pipe(Effect.mapError(serviceError("state"))), release: namespace.release.pipe(Effect.mapError(serviceError("release"))), launch: (generation, inputs, candidate) => From 67bef7cc175a7627627721eb017d31ae1ba0d020 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 17:03:29 +0200 Subject: [PATCH 050/142] fix(stack): abandon a stack whose registration disappears during an HTTP-requested stop --- ...ost.container-shutdown.integration.test.ts | 211 ++++++++++-------- packages/stack/src/StackHost.ts | 19 +- 2 files changed, 126 insertions(+), 104 deletions(-) diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index 9d6e868f6e..82418da3b7 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -26,6 +26,7 @@ import { ownerClient, ownerExitProbe, waitForOwnerExit, + type HostAccess, } from "./HostProcess.ts"; import { makeContainerRuntime, resolveEngineTarget } from "./runtime/Container.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; @@ -519,106 +520,126 @@ it.live.skipIf(process.platform === "win32")( { timeout: 180_000 }, ); -it.live.skipIf(process.platform === "win32")( - "abandons a stack whose registration disappears while a signal-driven stop is still draining", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-abandon-gated-" }); - const stackId = `abandon-gated-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; - const dataRoot = yield* makeDockerDatabaseRoot("stack-abandon-gated-data-", stackId).pipe( - Effect.flatMap(fs.realPath), - ); - const stateRoot = path.dirname(path.dirname(dataRoot)); - const cacheRoot = `${base}/cache`; - const gateDir = `${base}/gate`; - yield* fs.makeDirectory(gateDir, { recursive: true }); - const state = yield* stateFor(stateRoot); - yield* state.save({ - id: stackId, - runtime: "docker", - identity: { - projectRoot: `${base}/project`, - branchContext: "abandon-gated-test", - stackName: stackId, - }, - instances: [], - lifetime: "detached", - composition: { members: [], dependencies: [] }, - ports: [], - }); - yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); - const access = yield* launchHost(state, { - stateRoot, - cacheRoot, - stackId, - entrypoint: gatedDrainFixture, - entrypointArgs: [gateDir], - }); - const client = yield* ownerClient(access); - const database = yield* client.createService({ - service: "database", - config: { - version: "17", - databasePassword: Redacted.make("abandon-gated-password"), - jwtSecret: Redacted.make("abandon-gated-jwt-secret-at-least-thirty-two-characters"), - jwtExpiry: 3600, - }, - endpoints: { sql: { port: "auto" } }, - }); - yield* client.startService({ id: database.id }); - yield* client.readyService({ id: database.id }); - const status = yield* client.status({ id: database.id }); - const port = status.endpoints.find((endpoint) => endpoint.name === "sql")?.port; - if (port === undefined) return yield* Effect.die("Missing database sql endpoint"); - const containerEnvRoot = `${dataRoot}/.container-env`; - expect( - yield* fs.exists(containerEnvRoot), - "the stack's shared container-env scratch directory exists", - ).toBe(true); +type StopTrigger = "signal" | "http"; - // Keeps the sql listener's outstanding-connection count above zero for the whole gated - // window, so drain can only resolve through the deadline this test controls, never - // through every connection reaching zero on its own. - const held = yield* Effect.acquireRelease( - Effect.callback((resume) => { - const connection = Net.createConnection({ host: "127.0.0.1", port }); - connection.once("connect", () => resume(Effect.succeed(connection))); - connection.once("error", (cause) => resume(Effect.die(cause))); - return Effect.sync(() => connection.destroy()); - }), - (connection) => Effect.sync(() => connection.destroy()), - ); +/** + * Starts a data-preserving stop through `trigger` without waiting for it to finish: a + * signal-driven stop runs in the owner process on its own, while an HTTP-driven stop is forked + * so the gated drain it blocks on never blocks the caller. + */ +const triggerStop = (trigger: StopTrigger, access: HostAccess) => + trigger === "signal" + ? Effect.sync(() => process.kill(access.endpoint.pid, "SIGTERM")) + : Effect.forkScoped(shutdownOwner(access, false).pipe(Effect.ignore)); - // Subscribes to the drain-deadline wait's own entry marker before sending the signal that - // triggers it, so the registration deletion below never races the gate itself. - const waiting = yield* watchEntry(gateDir, "waiting", true); - process.kill(access.endpoint.pid, "SIGTERM"); - yield* waiting.pipe(Effect.timeout("30 seconds")); +const abandonsWhileStopDrains = (trigger: StopTrigger) => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-abandon-gated-" }); + const stackId = `abandon-gated-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const dataRoot = yield* makeDockerDatabaseRoot("stack-abandon-gated-data-", stackId).pipe( + Effect.flatMap(fs.realPath), + ); + const stateRoot = path.dirname(path.dirname(dataRoot)); + const cacheRoot = `${base}/cache`; + const gateDir = `${base}/gate`; + yield* fs.makeDirectory(gateDir, { recursive: true }); + const state = yield* stateFor(stateRoot); + yield* state.save({ + id: stackId, + runtime: "docker", + identity: { + projectRoot: `${base}/project`, + branchContext: "abandon-gated-test", + stackName: stackId, + }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + ports: [], + }); + yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); + const access = yield* launchHost(state, { + stateRoot, + cacheRoot, + stackId, + entrypoint: gatedDrainFixture, + entrypointArgs: [gateDir], + }); + const client = yield* ownerClient(access); + const database = yield* client.createService({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("abandon-gated-password"), + jwtSecret: Redacted.make("abandon-gated-jwt-secret-at-least-thirty-two-characters"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* client.startService({ id: database.id }); + yield* client.readyService({ id: database.id }); + const status = yield* client.status({ id: database.id }); + const port = status.endpoints.find((endpoint) => endpoint.name === "sql")?.port; + if (port === undefined) return yield* Effect.die("Missing database sql endpoint"); + const containerEnvRoot = `${dataRoot}/.container-env`; + expect( + yield* fs.exists(containerEnvRoot), + "the stack's shared container-env scratch directory exists", + ).toBe(true); - yield* fs.remove(`${stateRoot}/${stackId}/state.json`); - yield* fs.writeFileString(`${gateDir}/release`, ""); - yield* Effect.sync(() => held.destroy()); + // Keeps the sql listener's outstanding-connection count above zero for the whole gated + // window, so drain can only resolve through the deadline this test controls, never + // through every connection reaching zero on its own. + const held = yield* Effect.acquireRelease( + Effect.callback((resume) => { + const connection = Net.createConnection({ host: "127.0.0.1", port }); + connection.once("connect", () => resume(Effect.succeed(connection))); + connection.once("error", (cause) => resume(Effect.die(cause))); + return Effect.sync(() => connection.destroy()); + }), + (connection) => Effect.sync(() => connection.destroy()), + ); - yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( - Effect.retry({ while: hasReason("owner-exit-pending") }), - Effect.timeout("30 seconds"), - ); + // Subscribes to the drain-deadline wait's own entry marker before triggering the stop, so + // the registration deletion below never races the gate itself. + const waiting = yield* watchEntry(gateDir, "waiting", true); + yield* triggerStop(trigger, access); + yield* waiting.pipe(Effect.timeout("30 seconds")); - expect( - yield* fs.exists(`${stateRoot}/${stackId}/state.json`), - "no registration is republished", - ).toBe(false); - expect( - yield* fs.exists(containerEnvRoot), - "the registration lost during the gated stop still reaches abandonment's cleanup", - ).toBe(false); - expect(yield* containers(stackId, dataRoot), "no containers are left behind").toEqual([]); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + yield* fs.remove(`${stateRoot}/${stackId}/state.json`); + yield* fs.writeFileString(`${gateDir}/release`, ""); + yield* Effect.sync(() => held.destroy()); + + yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.retry({ while: hasReason("owner-exit-pending") }), + Effect.timeout("30 seconds"), + ); + + expect( + yield* fs.exists(`${stateRoot}/${stackId}/state.json`), + "no registration is republished", + ).toBe(false); + expect( + yield* fs.exists(containerEnvRoot), + "the registration lost during the gated stop still reaches abandonment's cleanup", + ).toBe(false); + expect(yield* containers(stackId, dataRoot), "no containers are left behind").toEqual([]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))); + +it.live.skipIf(process.platform === "win32")( + "abandons a stack whose registration disappears while a signal-driven stop is still draining", + () => abandonsWhileStopDrains("signal"), + { timeout: 180_000 }, +); + +it.live.skipIf(process.platform === "win32")( + "abandons a stack whose registration disappears while an HTTP-driven stop is still draining", + () => abandonsWhileStopDrains("http"), { timeout: 180_000 }, ); diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index aa445cf167..fdcbe2886b 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -672,7 +672,6 @@ export const runStackHost = Effect.fn("StackHost.run")( Effect.map(() => "done" as const), Effect.raceFirst(Queue.take(events)), ); - if (event === "done") break; if (event === "creator-gone") { yield* started.shutdown(true).pipe( Effect.tapCause((cause) => Effect.logError("Session stack destroy failed", cause)), @@ -684,14 +683,16 @@ export const runStackHost = Effect.fn("StackHost.run")( yield* started.abandon; break; } - const shutdownSucceeded = yield* started.shutdown(false).pipe( - Effect.tapCause((cause) => Effect.logError("Stack shutdown failed", cause)), - Effect.matchCause({ onSuccess: () => true, onFailure: () => false }), - ); - if (!shutdownSucceeded) continue; - // A registration deletion that arrives while this data-preserving stop is already - // running only reaches `events` as a queued, now-unread `registration-gone`: a direct - // re-read through the same state API the poll uses catches it before the owner exits. + if (event !== "done") { + const shutdownSucceeded = yield* started.shutdown(false).pipe( + Effect.tapCause((cause) => Effect.logError("Stack shutdown failed", cause)), + Effect.matchCause({ onSuccess: () => true, onFailure: () => false }), + ); + if (!shutdownSucceeded) continue; + } + // A registration deletion that arrives while a data-preserving stop is already running + // only reaches `events` as a queued, now-unread `registration-gone`: a direct re-read + // through the same state API the poll uses catches it before the owner exits. if (yield* registrationConfirmedGone(state, id)) yield* started.abandon; break; } From c357fac94c49986b245209a8ec169fbe44335b60 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 17:53:43 +0200 Subject: [PATCH 051/142] test(stack): assert a destroyed stack's state and data are gone instead of its directory --- .../experimental/stack/start/start.e2e.test.ts | 12 +++++++++--- packages/stack/tests/whole-stack/scenarios.ts | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts b/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts index fca1046b9b..616b4f6f11 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts @@ -186,7 +186,8 @@ describe("stack start (compiled e2e)", () => { const cleanup = Effect.gen(function* () { let cleanupComplete = stackDestroyed; if (!cleanupComplete && home !== undefined) { - const candidates = yield* readDirectory(join(home.dir, "stacks")).pipe( + const stacksDir = join(home.dir, "stacks"); + const candidates = yield* readDirectory(stacksDir).pipe( Effect.catchIf( (cause) => Predicate.isTagged(cause, "PlatformError") && @@ -194,7 +195,10 @@ describe("stack start (compiled e2e)", () => { () => Effect.succeed([]), ), ); - const discovered = candidates.filter((entry) => /^[0-9a-f]{64}$/u.test(entry)); + const discovered = yield* Effect.filter( + candidates.filter((entry) => /^[0-9a-f]{64}$/u.test(entry)), + (entry) => access(join(stacksDir, entry, "state.json")).pipe(Effect.isSuccess), + ); const ownedId = stackId ?? (discovered.length === 1 ? discovered[0] : undefined); if (ownedId !== undefined) { yield* destroyStack(home.dir, ownedId); @@ -549,7 +553,9 @@ describe("stack start (compiled e2e)", () => { yield* destroyStack(homeDir.dir, idText); stackDestroyed = true; - const destroyed = yield* Effect.exit(access(join(homeDir.dir, "stacks", idText))); + const destroyed = yield* Effect.exit( + access(join(homeDir.dir, "stacks", idText, "state.json")), + ); expect(Exit.isFailure(destroyed)).toBe(true); }), ), diff --git a/packages/stack/tests/whole-stack/scenarios.ts b/packages/stack/tests/whole-stack/scenarios.ts index 19430852fe..53b3f826eb 100644 --- a/packages/stack/tests/whole-stack/scenarios.ts +++ b/packages/stack/tests/whole-stack/scenarios.ts @@ -133,7 +133,7 @@ const assertOwnedPathsGone = Effect.fn("WholeStack.assertOwnedPathsGone")((fixtu expect(yield* fs.exists(databasePath)).toBe(false); expect(yield* fs.exists(functionsPath)).toBe(false); expect(yield* fs.exists(statePath)).toBe(false); - expect(yield* fs.exists(`${fixture.locations.stateRoot}/${fixture.stack.id}`)).toBe(false); + expect((yield* fs.exists(dataRoot)) ? yield* fs.readDirectory(dataRoot) : []).toEqual([]); }), ); From f2850f0c6c2f3b7720680d84b220cea84c5aafd5 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 18:55:43 +0200 Subject: [PATCH 052/142] test(cli): assert registered stacks after db diff instead of stack directories --- .../cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts b/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts index 37603ac7da..418d70771c 100644 --- a/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts +++ b/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts @@ -2,7 +2,7 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Path, Redacted } from "effect"; import { FetchHttpClient } from "effect/unstable/http"; -import { create as createStack } from "@supabase/stack/effect"; +import { create as createStack, discover as discoverStacks } from "@supabase/stack/effect"; import { tmpdir } from "node:os"; import { runSupabaseEffect } from "../../../../tests/helpers/cli.ts"; import { destroyTestStack } from "../../../../tests/helpers/stack-cleanup.ts"; @@ -130,10 +130,8 @@ describe("supabase db diff (stack shadow baseline cache)", () => { expect(second.stdout).toMatch(/stack_cache_second/iu); expect(second.stdout).not.toMatch(/stack_cache_first/iu); - const stackEntries = (yield* fs.readDirectory(path.join(home, "stacks"))).filter( - (entry) => /^[0-9a-f]{64}$/u.test(entry), - ); - expect(stackEntries).toEqual([stack.id]); + const registered = yield* discoverStacks({ stateRoot: path.join(home, "stacks") }); + expect(registered.map(({ definition }) => definition.id)).toEqual([stack.id]); }).pipe(Effect.scoped, Effect.provide([BunServices.layer, FetchHttpClient.layer])), { timeout: TEST_TIMEOUT_MS }, ); From eca69d75b042fb97f7e2fa3d6723ab9b631bfc63 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 20:51:10 +0200 Subject: [PATCH 053/142] fix(stack): drain HTTP listeners without waiting on idle keep-alive connections --- .../stack/src/HttpProxy.integration.test.ts | 40 ++--- packages/stack/src/HttpProxy.ts | 33 ++-- .../stack/src/Network.integration.test.ts | 144 ++++++++++++++++++ packages/stack/src/Network.ts | 4 + 4 files changed, 188 insertions(+), 33 deletions(-) diff --git a/packages/stack/src/HttpProxy.integration.test.ts b/packages/stack/src/HttpProxy.integration.test.ts index 02f4568e45..abca735b7d 100644 --- a/packages/stack/src/HttpProxy.integration.test.ts +++ b/packages/stack/src/HttpProxy.integration.test.ts @@ -1,6 +1,6 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Data, Deferred, Effect, Fiber, Layer, Stream, SubscriptionRef } from "effect"; +import { Data, Deferred, Effect, Fiber, Layer, SubscriptionRef } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { createServer, type Server, type ServerResponse } from "node:http"; // oxlint-disable-line effecttsgo/node-builtin-import -- raw server fixture. import { createServer as createTcpServer, Socket, type Server as NetServer } from "node:net"; // oxlint-disable-line effecttsgo/node-builtin-import -- raw disconnect fixture. @@ -1209,15 +1209,11 @@ it.live( ); it.live( - "refuses a new connection after stopAccepting while an established connection keeps completing requests until it closes", + "closes an idle established connection once stopAccepting runs, while still refusing new connections", () => Effect.scoped( Effect.gen(function* () { - let requests = 0; - const backend = createServer((_request, response) => { - requests += 1; - response.end(requests === 1 ? "first-response" : "second-response"); - }); + const backend = createServer((_request, response) => response.end("first-response")); const backendAddress = yield* listen(backend); const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); yield* proxy.setRoutes([ @@ -1225,29 +1221,24 @@ it.live( ]); const client = yield* connectRaw(proxy.port, proxy.host); - client.write("GET / HTTP/1.1\r\nHost: localhost\r\n\r\n"); + client.write("GET / HTTP/1.1\r\nHost: localhost\r\nConnection: keep-alive\r\n\r\n"); const first = yield* readFullResponse(client); expect(first.status).toBe(200); expect(first.body).toBe("first-response"); - expect(yield* SubscriptionRef.get(proxy.outstandingConnections)).toBe(1); + // The request already finished, so the connection sits idle with no active work. + expect(yield* SubscriptionRef.get(proxy.outstandingConnections)).toBe(0); + + const closed = yield* Effect.callback((resume) => { + const onClose = () => resume(Effect.void); + client.once("close", onClose); + if (client.destroyed) onClose(); + return Effect.sync(() => client.off("close", onClose)); + }).pipe(Effect.timeout("5 seconds"), Effect.forkScoped); yield* proxy.stopAccepting; expect(yield* connectionRefused(proxy.port, proxy.host)).toBe(true); - - client.write("GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"); - const second = yield* readFullResponse(client); - expect(second.status).toBe(200); - expect(second.body).toBe("second-response"); - - const reachedZero = yield* Stream.runHead( - Stream.filter( - SubscriptionRef.changes(proxy.outstandingConnections), - (count) => count === 0, - ), - ).pipe(Effect.forkScoped); - yield* Effect.sync(() => client.end()); - yield* Fiber.join(reachedZero); + yield* Fiber.join(closed); }), ).pipe(Effect.provide(NodeServices.layer)), ); @@ -1265,7 +1256,8 @@ it.live("destroys every established connection immediately when cutAll runs", () const response = yield* readFullResponse(client); expect(response.status).toBe(200); expect(response.body).toBe("ok"); - expect(yield* SubscriptionRef.get(proxy.outstandingConnections)).toBe(1); + // The request already finished, so the connection sits idle with no active work. + expect(yield* SubscriptionRef.get(proxy.outstandingConnections)).toBe(0); const closed = yield* Effect.callback((resume) => { const onClose = () => resume(Effect.void); diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index e280d72033..872e29e798 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -49,7 +49,10 @@ export interface HttpProxy { readonly stopAccepting: Effect.Effect; /** Reverses `stopAccepting`: recovers a listener left refusing after a failed drain. */ readonly resumeAccepting: Effect.Effect; - /** The count of established client connections, observable until it reaches 0. */ + /** + * The count of connections with active work, observable until it reaches 0: a request in + * flight or an upgraded socket, never an idle keep-alive connection. + */ readonly outstandingConnections: SubscriptionRef.SubscriptionRef; /** Destroys every established connection immediately. */ readonly cutAll: Effect.Effect; @@ -432,10 +435,16 @@ export const makeHttpProxy = (options: { const services = yield* Effect.context(); const sockets = new Set(); const outstandingConnections = yield* SubscriptionRef.make(0); - // `server.close()` also tears down idle keep-alive sockets immediately, which would end - // established connections instead of letting them keep flowing; track acceptance separately. + // `server.close()` also tears down active connections immediately, not just idle ones, so + // draining tracks acceptance separately and closes idle sockets explicitly instead. let accepting = true; + const closeNowIdle = () => { + if (!accepting) server.closeIdleConnections(); + }; const server = createServer((request, response) => { + Effect.runSyncWith(services)( + SubscriptionRef.update(outstandingConnections, (count) => count + 1), + ); runRequest( Effect.scoped( Effect.gen(function* () { @@ -470,6 +479,12 @@ export const makeHttpProxy = (options: { ); } }), + ).pipe( + Effect.ensuring( + SubscriptionRef.update(outstandingConnections, (count) => count - 1).pipe( + Effect.andThen(Effect.sync(closeNowIdle)), + ), + ), ), ); }); @@ -479,18 +494,15 @@ export const makeHttpProxy = (options: { return; } sockets.add(socket); - Effect.runSyncWith(services)( - SubscriptionRef.update(outstandingConnections, (count) => count + 1), - ); socket.once("close", () => { sockets.delete(socket); - Effect.runSyncWith(services)( - SubscriptionRef.update(outstandingConnections, (count) => count - 1), - ); }); }); server.on("upgrade", (request, socket, head) => { socket.on("error", () => socket.destroy()); + Effect.runSyncWith(services)( + SubscriptionRef.update(outstandingConnections, (count) => count + 1), + ); runRequest( Effect.scoped( Effect.gen(function* () { @@ -508,6 +520,8 @@ export const makeHttpProxy = (options: { Effect.catch(() => Effect.sync(() => socket.destroy())), ); }), + ).pipe( + Effect.ensuring(SubscriptionRef.update(outstandingConnections, (count) => count - 1)), ), ); }); @@ -539,6 +553,7 @@ export const makeHttpProxy = (options: { ), stopAccepting: Effect.sync(() => { accepting = false; + server.closeIdleConnections(); }), resumeAccepting: Effect.sync(() => { accepting = true; diff --git a/packages/stack/src/Network.integration.test.ts b/packages/stack/src/Network.integration.test.ts index 9ca7022628..3e40e059b2 100644 --- a/packages/stack/src/Network.integration.test.ts +++ b/packages/stack/src/Network.integration.test.ts @@ -630,3 +630,147 @@ it.live("addresses docker runtime endpoints through the stack host alias", () => }), ).pipe(Effect.provide(NodeServices.layer)), ); + +it.live("drains past an idle HTTP keep-alive connection without ever reaching the deadline", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-drain-idle-" }); + const state = yield* makeTestState(root); + yield* state.save(stack("stack", "auto")); + const target = yield* backend; + const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); + const service = yield* network.register({ + id: "studio", + endpoints: { http: endpoint(target, Effect.succeed(true)) }, + }); + yield* service.bind; + const address = yield* service.address("http", "host"); + + // A served keep-alive request proves the listener accepted and tracks this socket, so its + // closure is observed directly instead of by reprobing the port. + const socket = yield* Effect.callback((resume) => { + const connection = Net.createConnection({ host: address.host, port: address.port }); + connection.once("connect", () => { + connection.write( + `GET /hello HTTP/1.1\r\nHost: ${address.host}:${address.port}\r\nConnection: keep-alive\r\n\r\n`, + ); + }); + connection.once("data", () => resume(Effect.succeed(connection))); + connection.on("error", (cause) => + resume(Effect.fail(new FixtureError({ message: cause.message }))), + ); + return Effect.sync(() => connection.destroy()); + }); + const closed = yield* Effect.callback((resume) => { + socket.once("close", () => resume(Effect.void)); + return Effect.void; + }).pipe(Effect.forkChild); + + // A deadline that never fires proves drain resolved through quiescence alone: if the idle + // connection still counted as outstanding, this would hang instead of completing. + yield* network.drain.pipe( + Effect.provideService(Network.ShutdownDrainDeadline, Effect.never), + Effect.timeout("5 seconds"), + ); + yield* Fiber.join(closed).pipe(Effect.timeout("2 seconds")); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live( + "lets an in-flight HTTP request finish during drain, then closes its now-idle connection", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-drain-inflight-" }); + const state = yield* makeTestState(root); + yield* state.save(stack("stack", "auto")); + const services = yield* Effect.context(); + const requested = yield* Deferred.make(); + const release = yield* Deferred.make(); + const held = Effect.acquireRelease( + Effect.callback< + { host: string; port: number; server: ReturnType }, + FixtureError + >((resume) => { + const server = createServer((_request, response) => { + Effect.runForkWith(services)( + Deferred.succeed(requested, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.andThen(Effect.sync(() => response.end("backend:held"))), + ), + ); + }); + const onError = (cause: Error) => + resume(Effect.fail(new FixtureError({ message: cause.message }))); + server.once("error", onError); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (address === null || typeof address === "string") + resume(Effect.fail(new FixtureError({ message: "no address" }))); + else resume(Effect.succeed({ host: "127.0.0.1", port: address.port, server })); + }); + return Effect.sync(() => { + server.off("error", onError); + server.close(); + }); + }), + ({ server }) => + Effect.callback((resume) => { + server.close(() => resume(Effect.void)); + return Effect.void; + }), + ); + const target = yield* held; + const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); + const service = yield* network.register({ + id: "studio", + endpoints: { http: endpoint(target, Effect.succeed(true)) }, + }); + yield* service.bind; + const address = yield* service.address("http", "host"); + + let body = ""; + const received = yield* Deferred.make(); + const socket = yield* Effect.callback((resume) => { + const connection = Net.createConnection({ host: address.host, port: address.port }); + connection.once("connect", () => { + connection.write( + `GET /hello HTTP/1.1\r\nHost: ${address.host}:${address.port}\r\nConnection: keep-alive\r\n\r\n`, + ); + }); + connection.on("data", (bytes) => { + body += bytes.toString(); + if (body.includes("backend:held")) + Effect.runForkWith(services)(Deferred.succeed(received, undefined)); + }); + connection.once("connect", () => resume(Effect.succeed(connection))); + connection.on("error", (cause) => + resume(Effect.fail(new FixtureError({ message: cause.message }))), + ); + return Effect.sync(() => connection.destroy()); + }); + const closed = yield* Effect.callback((resume) => { + socket.once("close", () => resume(Effect.void)); + return Effect.void; + }).pipe(Effect.forkChild); + yield* Deferred.await(requested).pipe(Effect.timeout("2 seconds")); + + // A deadline that never fires proves drain is genuinely waiting on the request in flight, + // not racing a timer, while the held response keeps it from resolving prematurely. + const drained = yield* network.drain.pipe( + Effect.provideService(Network.ShutdownDrainDeadline, Effect.never), + Effect.forkChild, + ); + yield* Deferred.succeed(release, undefined); + // Waiting for the response body itself, not just the drain fiber, proves completion is + // observed through the client's own socket rather than assumed once the fiber joins. + yield* Deferred.await(received).pipe(Effect.timeout("5 seconds")); + yield* Fiber.join(drained).pipe(Effect.timeout("5 seconds")); + expect(body).toContain("backend:held"); + yield* Fiber.join(closed).pipe(Effect.timeout("2 seconds")); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index 274a218944..b2834fc0de 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -32,6 +32,10 @@ export const ShutdownDrainDeadline = Context.Reference>( interface ListenerHandle { readonly stopAccepting: Effect.Effect; readonly resumeAccepting: Effect.Effect; + /** + * What counts as active is listener-specific: every TCP connection, or an HTTP connection only + * while it has a request in flight or is upgraded, excluding idle keep-alive sockets. + */ readonly outstandingConnections: SubscriptionRef.SubscriptionRef; readonly cutAll: Effect.Effect; } From 1299c5c4c72fa52a3256cca3eea85e05ba561e4a Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Oct 2026 23:47:24 +0200 Subject: [PATCH 054/142] chore(cli): move the functions deploy changes off the stack branch Develop already converted functions deploy to Effect; the bundle directory cleanup fix moves to its own branch based on develop. --- .oxlintrc.effect.json | 1 - .../deploy/deploy.integration.test.ts | 575 +----- .../cli/src/shared/functions/deploy.errors.ts | 118 +- apps/cli/src/shared/functions/deploy.ts | 1684 ++++++++--------- .../src/shared/functions/deploy.unit.test.ts | 62 +- apps/cli/src/shared/functions/serve.ts | 4 +- .../tests/helpers/child-process-spawner.ts | 9 +- 7 files changed, 859 insertions(+), 1594 deletions(-) diff --git a/.oxlintrc.effect.json b/.oxlintrc.effect.json index 2f9ab05d35..137f3fb227 100644 --- a/.oxlintrc.effect.json +++ b/.oxlintrc.effect.json @@ -12,7 +12,6 @@ "!apps/cli/src/shared/compute/**", "!apps/cli/src/shared/config/**", "!apps/cli/src/shared/feedback/**", - "!apps/cli/src/shared/functions/deploy.ts", "!apps/cli/src/shared/functions/functions-docker.ts", "!apps/cli/src/shared/functions/functions-docker.unit.test.ts", "!apps/cli/src/shared/functions/serve.ts", diff --git a/apps/cli/src/commands/functions/deploy/deploy.integration.test.ts b/apps/cli/src/commands/functions/deploy/deploy.integration.test.ts index 20358f03f3..600907e826 100644 --- a/apps/cli/src/commands/functions/deploy/deploy.integration.test.ts +++ b/apps/cli/src/commands/functions/deploy/deploy.integration.test.ts @@ -1,27 +1,6 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { - Clock, - Deferred, - Duration, - Effect, - Exit, - Fiber, - FileSystem, - Layer, - Option, - Path, - Predicate, - Queue, - Ref, - Schema, - Sink, - Stdio, - Stream, -} from "effect"; -import { ChildProcessSpawner } from "effect/unstable/process"; -import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; -import * as TestClock from "effect/testing/TestClock"; +import { Effect, Exit, FileSystem, Layer, Option, Path, Stdio } from "effect"; import { YesFlag } from "../../../command-internal/global-flags.ts"; import { stripControlSequences } from "../../../shared/output/strip-control-sequences.ts"; @@ -42,11 +21,9 @@ import { shouldChmodBundleOutputDirectory, } from "../../../shared/functions/deploy.ts"; import { toDockerPath } from "../../../shared/functions/functions-docker.ts"; -import { FunctionsApiStatusError } from "../../../shared/functions/functions-api.errors.ts"; import { functionsGoConfigCompat } from "../../../command-internal/functions-go-config.ts"; import { ConflictingFunctionDeployFlagsError, - FunctionAssetOutsideRootError, InvalidFunctionDeploySlugError, NoFunctionsToDeployError, } from "../../../shared/functions/deploy.errors.ts"; @@ -56,7 +33,6 @@ import { functionsDeploy } from "./deploy.handler.ts"; import type { FunctionsDeployFlags } from "./deploy.command.ts"; const tempRoot = useTempWorkdir("supabase-functions-deploy-legacy-"); -const isRoot = typeof process.getuid === "function" && process.getuid() === 0; const baseFlags: FunctionsDeployFlags = { functionNames: ["hello-world"], @@ -847,7 +823,7 @@ describe("functions deploy", () => { const error = yield* functionsDeploy({ ...baseFlags, useDocker: true }).pipe(Effect.flip); expect(error).toBeInstanceOf(ConflictingFunctionDeployFlagsError); - if (!Schema.is(ConflictingFunctionDeployFlagsError)(error)) { + if (!(error instanceof ConflictingFunctionDeployFlagsError)) { throw new Error(`unexpected error: ${String(error)}`); } expect(error.message).toBe( @@ -1155,7 +1131,7 @@ describe("functions deploy", () => { const error = yield* deployNoFunctions.pipe(Effect.flip); expect(error).toBeInstanceOf(NoFunctionsToDeployError); - if (!Schema.is(NoFunctionsToDeployError)(error)) { + if (!(error instanceof NoFunctionsToDeployError)) { throw new Error(`unexpected error: ${String(error)}`); } expect(error.message).toBe( @@ -1672,549 +1648,4 @@ describe("functions deploy", () => { }).pipe(Effect.provide(layer), Effect.ensuring(removeTempRoot)); }); }); - - describe("bundle temp directory cleanup", () => { - const readBundleOutputDirs = Effect.fnUntraced(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const entries = yield* fs.readDirectory(path.join(tempRoot.current, "supabase", ".temp")); - return entries.filter((entry) => entry.startsWith(".supabase-output-")); - }); - - /** Spawns `docker run` as a process that never exits until its fiber is interrupted. */ - function mockDockerRunHangingSpawner(onRunStarted: () => void) { - let nextPid = 0; - return Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make((command) => { - const cmd = Predicate.isTagged(command, "StandardCommand") ? command.command : ""; - const args = Predicate.isTagged(command, "StandardCommand") ? command.args : []; - const isRun = cmd === "docker" && args[0] === "run"; - nextPid += 1; - if (isRun) { - onRunStarted(); - } - return Effect.succeed( - ChildProcessSpawner.makeHandle({ - pid: ChildProcessSpawner.ProcessId(9000 + nextPid), - stdout: Stream.empty, - stderr: Stream.empty, - all: Stream.empty, - exitCode: isRun ? Effect.never : Effect.succeed(ChildProcessSpawner.ExitCode(0)), - isRunning: Effect.succeed(true), - stdin: Sink.drain, - kill: () => Effect.void, - unref: Effect.succeed(Effect.void), - getInputFd: () => Sink.drain, - getOutputFd: () => Stream.empty, - }), - ); - }), - ); - } - - it.live("removes the bundle temp directory when the bundler container exits non-zero", () => { - const out = mockOutput({ format: "text" }); - const api = mockCommandPlatformApi({ - handler: (request) => Effect.succeed(jsonResponse(request, 200, [])), - }); - const child = mockChildProcessSpawner({ - exitCode: (record) => (record.command === "docker" && record.args[0] === "run" ? 1 : 0), - }); - const layer = Layer.mergeAll( - buildTestRuntime({ - out, - api, - cliSettings: mockCommandSettings({ workdir: tempRoot.current }), - runtimeInfo: mockRuntimeInfo({ cwd: tempRoot.current }), - }), - Layer.succeed(YesFlag, false), - child.layer, - Stdio.layerTest({ - args: Effect.succeed(["functions", "deploy", "hello-world", "--use-api=false"]), - }), - ); - - return Effect.gen(function* () { - yield* writeCliConfig(tempRoot.current); - yield* writeLocalFunction(tempRoot.current, "hello-world"); - - const error = yield* functionsDeploy({ - ...baseFlags, - useApi: false, - useDocker: true, - }).pipe(Effect.flip); - - expect(error).toBeInstanceOf(Error); - expect(error).toMatchObject({ message: "failed to bundle function: exit 1" }); - expect(yield* readBundleOutputDirs()).toEqual([]); - }).pipe(Effect.provide(layer), Effect.ensuring(removeTempRoot)); - }); - - it.live("removes the bundle temp directory when the deploy is interrupted mid-bundle", () => { - const out = mockOutput({ format: "text" }); - const api = mockCommandPlatformApi({ - handler: (request) => Effect.succeed(jsonResponse(request, 200, [])), - }); - const bundlerStarted = Deferred.makeUnsafe(); - const child = mockDockerRunHangingSpawner(() => { - Deferred.doneUnsafe(bundlerStarted, Exit.void); - }); - const layer = Layer.mergeAll( - buildTestRuntime({ - out, - api, - cliSettings: mockCommandSettings({ workdir: tempRoot.current }), - runtimeInfo: mockRuntimeInfo({ cwd: tempRoot.current }), - }), - Layer.succeed(YesFlag, false), - child, - Stdio.layerTest({ - args: Effect.succeed(["functions", "deploy", "hello-world", "--use-api=false"]), - }), - ); - - return Effect.gen(function* () { - yield* writeCliConfig(tempRoot.current); - yield* writeLocalFunction(tempRoot.current, "hello-world"); - - const fiber = yield* functionsDeploy({ - ...baseFlags, - useApi: false, - useDocker: true, - }).pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(bundlerStarted); - yield* Fiber.interrupt(fiber); - - expect(yield* readBundleOutputDirs()).toEqual([]); - }).pipe(Effect.provide(layer), Effect.ensuring(removeTempRoot)); - }); - }); - - it.live( - "does not retry a create response whose success status decodes to a malformed body", - () => { - const out = mockOutput({ format: "text" }); - let createCalls = 0; - const api = mockCommandPlatformApi({ - handler: (request) => { - if (request.method === "GET") { - return Effect.succeed(jsonResponse(request, 200, [])); - } - createCalls += 1; - return Effect.succeed( - HttpClientResponse.fromWeb( - request, - new Response("not valid json", { - status: 201, - headers: { "content-type": "application/json" }, - }), - ), - ); - }, - }); - const child = mockDockerBundleSpawner(); - const layer = Layer.mergeAll( - buildTestRuntime({ - out, - api, - cliSettings: mockCommandSettings({ workdir: tempRoot.current }), - runtimeInfo: mockRuntimeInfo({ cwd: tempRoot.current }), - }), - Layer.succeed(YesFlag, false), - child.layer, - Stdio.layerTest({ - args: Effect.succeed(["functions", "deploy", "hello-world", "--use-api=false"]), - }), - ); - - return Effect.gen(function* () { - yield* writeCliConfig(tempRoot.current); - yield* writeLocalFunction(tempRoot.current, "hello-world"); - - const error = yield* functionsDeploy({ - ...baseFlags, - useApi: false, - useDocker: true, - }).pipe(Effect.flip); - - expect(error).toBeInstanceOf(FunctionsApiStatusError); - if (!(error instanceof FunctionsApiStatusError)) { - throw new Error(`unexpected error: ${String(error)}`); - } - expect(error.message).toContain("failed to read function response:"); - // Exactly one create request: a malformed success body is terminal, not retried. - expect(createCalls).toBe(1); - }).pipe(Effect.provide(layer), Effect.ensuring(removeTempRoot)); - }, - ); - - describe("transient API retries (Schedule-based backoff)", () => { - // `it.effect` (not `it.live`): the local filesystem setup below runs to completion with real - // I/O before `functionsDeploy` is forked, so nothing timed races against real I/O once - // `TestClock` takes over. The underlying `@supabase/api` client retries a transient response - // immediately (no delay — see `packages/api/src/internal/client.ts`'s `applySupabaseRetryPolicy`, - // `maxRetries` defaulting to 5, i.e. 6 raw requests per logical call), so each of - // `listRemoteFunctions`'s own 4 attempts produces 6 requests; only its own 1s/2s/4s - // `Schedule.exponential` backoff between attempts actually sleeps. - it.effect( - "gives up listing remote functions after exhausting the retry bound on persistent 5xx, " + - "honoring the exponential backoff and the API client's own transient retry", - () => - Effect.gen(function* () { - const out = mockOutput({ format: "text" }); - const requestCount = yield* Ref.make(0); - // Offered by the fake handler itself on every raw GET, so the test can wait on an - // observable signal instead of polling real time or guessing how many scheduler ticks - // a response takes to decode. - const requestSeen = yield* Queue.unbounded(); - const api = mockCommandPlatformApi({ - handler: (request) => { - if (request.method === "GET") { - return Ref.update(requestCount, (n) => n + 1).pipe( - Effect.andThen(Queue.offer(requestSeen, undefined)), - Effect.as(jsonResponse(request, 503, { message: "unavailable" })), - ); - } - return Effect.succeed(jsonResponse(request, 201, {})); - }, - }); - const layer = Layer.mergeAll( - buildTestRuntime({ - out, - api, - cliSettings: mockCommandSettings({ workdir: tempRoot.current }), - runtimeInfo: mockRuntimeInfo({ cwd: tempRoot.current }), - }), - Layer.succeed(YesFlag, false), - Stdio.layerTest({ - args: Effect.succeed(["functions", "deploy", "hello-world", "--use-api"]), - }), - ); - - yield* writeCliConfig(tempRoot.current).pipe(Effect.provide(layer)); - yield* writeLocalFunction(tempRoot.current, "hello-world").pipe(Effect.provide(layer)); - - // `TestClock.adjust` doesn't wait for the response decode, so advance only after the - // backoff sleep is registered: the immediate fork records it before it can suspend. - const sleepRegistered = yield* Queue.unbounded(); - const realClock = yield* Effect.clockWith(Effect.succeed); - const instrumentedClock: Clock.Clock = { - ...realClock, - sleep: (duration) => - Effect.gen(function* () { - const sleeping = yield* realClock - .sleep(duration) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Queue.offer(sleepRegistered, undefined); - yield* Fiber.join(sleeping); - }), - }; - - const fiber = yield* functionsDeploy(baseFlags).pipe( - Effect.flip, - Effect.provide(layer), - Effect.provideService(Clock.Clock, instrumentedClock), - Effect.forkChild({ startImmediately: true }), - ); - - // Waits for the API client's own 6-request transient retry to fully land before - // trusting `requestCount`: each `Queue.take` suspends until the handler's offer for - // that raw request has actually been processed, so this also drains the real, unmocked - // filesystem reads the config/discovery steps ahead of the first attempt perform — - // no real-timer polling or `TestClock` involved yet. - const awaitRequestBatch = Effect.all( - Array.from({ length: 6 }, () => Queue.take(requestSeen)), - { discard: true }, - ); - - yield* awaitRequestBatch; - expect(yield* Ref.get(requestCount)).toBe(6); - - // Each subsequent attempt is gated by `listRemoteFunctions`'s own exponential delay. - // `TestClock.adjust` only runs fibers whose wake time has been reached: stopping 1ms - // short of each deadline and asserting the count is still unchanged tells 1s/2s/4s - // backoff apart from any other interval (e.g. 0.5s/1s/2s would have already fired). - let expectedCount = 6; - for (const delayMs of [1_000, 2_000, 4_000]) { - yield* Queue.take(sleepRegistered); - yield* TestClock.adjust(Duration.millis(delayMs - 1)); - expect(yield* Ref.get(requestCount)).toBe(expectedCount); - yield* TestClock.adjust(Duration.millis(1)); - yield* awaitRequestBatch; - expectedCount += 6; - expect(yield* Ref.get(requestCount)).toBe(expectedCount); - } - expect(expectedCount).toBe(24); - - const error = yield* Fiber.join(fiber); - if (!(error instanceof FunctionsApiStatusError)) { - throw new Error(`unexpected error: ${String(error)}`); - } - expect(error.message).toBe( - 'unexpected list functions status 503: {"message":"unavailable"}', - ); - }).pipe(Effect.ensuring(removeTempRoot)), - ); - }); - - it.live("rejects a static file outside the source root with the existing message", () => { - const out = mockOutput({ format: "text" }); - const api = mockCommandPlatformApi({ - handler: (request) => Effect.succeed(jsonResponse(request, 200, [])), - }); - const layer = Layer.mergeAll( - buildTestRuntime({ - out, - api, - cliSettings: mockCommandSettings({ workdir: tempRoot.current }), - runtimeInfo: mockRuntimeInfo({ cwd: tempRoot.current }), - }), - Layer.succeed(YesFlag, false), - Stdio.layerTest({ - args: Effect.succeed(["functions", "deploy", "hello-world", "--use-api"]), - }), - ); - - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const outsideDir = yield* fs.makeTempDirectoryScoped({ prefix: "deploy-outside-" }); - const outsideFile = path.join(outsideDir, "secret.txt"); - yield* fs.writeFileString(outsideFile, "do not upload\n"); - - yield* writeCliConfig( - tempRoot.current, - `project_id = "test-project"\n\n[functions.hello-world]\nstatic_files = ["${outsideFile}"]\n`, - ); - yield* writeLocalFunction(tempRoot.current, "hello-world"); - - const error = yield* functionsDeploy(baseFlags).pipe(Effect.flip); - - expect(Schema.is(FunctionAssetOutsideRootError)(error)).toBe(true); - if (!Schema.is(FunctionAssetOutsideRootError)(error)) { - throw new Error(`unexpected error: ${String(error)}`); - } - expect(error.message).toBe(`refusing to upload asset outside source root: ${outsideFile}`); - }).pipe(Effect.provide(layer), Effect.scoped, Effect.ensuring(removeTempRoot)); - }); - - const functionDeployResponseBody = { - id: "function-id", - slug: "hello-world", - name: "hello-world", - status: "ACTIVE", - version: 2, - created_at: 1_687_423_025_152, - updated_at: 1_687_423_025_152, - verify_jwt: true, - import_map: true, - entrypoint_path: "functions/hello-world/index.ts", - import_map_path: "functions/hello-world/deno.json", - }; - - // A chmod 000 directory is a no-op under root, which would make the glob resolve normally - // instead of exercising the failure this test targets. - it.live.skipIf(isRoot)( - "warns with the real filesystem error and continues past an unreadable nested directory under a static_files glob", - () => { - const out = mockOutput({ format: "text" }); - const api = mockCommandPlatformApi({ - handler: (request) => { - if (request.method === "GET") { - return Effect.succeed(jsonResponse(request, 200, [])); - } - return Effect.succeed(jsonResponse(request, 201, functionDeployResponseBody)); - }, - }); - const layer = Layer.mergeAll( - buildTestRuntime({ - out, - api, - cliSettings: mockCommandSettings({ workdir: tempRoot.current }), - runtimeInfo: mockRuntimeInfo({ cwd: tempRoot.current }), - }), - Layer.succeed(YesFlag, false), - Stdio.layerTest({ - args: Effect.succeed(["functions", "deploy", "hello-world", "--use-api"]), - }), - ); - - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const staticRoot = path.join(tempRoot.current, "supabase", "static"); - const lockedDir = path.join(staticRoot, "locked"); - yield* fs.makeDirectory(lockedDir, { recursive: true }); - yield* fs.writeFileString(path.join(staticRoot, "visible.txt"), "visible\n"); - yield* fs.writeFileString(path.join(lockedDir, "secret.txt"), "secret\n"); - // Only the nested directory is unreadable — a reader can still list `static/` itself, so - // a bypassed chmod (not expected here, guarded by `skipIf(isRoot)`) would surface as an - // actual upload of both files rather than a silent pass. - yield* Effect.acquireRelease(fs.chmod(lockedDir, 0o000), () => - fs.chmod(lockedDir, 0o700).pipe(Effect.orDie), - ); - - yield* writeCliConfig( - tempRoot.current, - 'project_id = "test-project"\n\n[functions.hello-world]\nstatic_files = ["static/**/*.txt"]\n', - ); - yield* writeLocalFunction(tempRoot.current, "hello-world"); - - yield* functionsDeploy(baseFlags); - - expect(out.stderrText).toContain(`WARN: EACCES: permission denied, open '${lockedDir}'`); - expect(stripControlSequences(out.stdoutText)).toContain("Deployed Functions on project"); - }).pipe(Effect.provide(layer), Effect.scoped, Effect.ensuring(removeTempRoot)); - }, - ); - - it.live( - "warns and continues past a static_files glob that is not a valid regular expression", - () => { - const out = mockOutput({ format: "text" }); - const api = mockCommandPlatformApi({ - handler: (request) => { - if (request.method === "GET") { - return Effect.succeed(jsonResponse(request, 200, [])); - } - return Effect.succeed(jsonResponse(request, 201, functionDeployResponseBody)); - }, - }); - const layer = Layer.mergeAll( - buildTestRuntime({ - out, - api, - cliSettings: mockCommandSettings({ workdir: tempRoot.current }), - runtimeInfo: mockRuntimeInfo({ cwd: tempRoot.current }), - }), - Layer.succeed(YesFlag, false), - Stdio.layerTest({ - args: Effect.succeed(["functions", "deploy", "hello-world", "--use-api"]), - }), - ); - - return Effect.gen(function* () { - // `[z-a]` is a glob bracket expression with an out-of-order range: valid Deno/Go glob - // syntax, but `new RegExp(...)` rejects it as an invalid character class. - yield* writeCliConfig( - tempRoot.current, - 'project_id = "test-project"\n\n[functions.hello-world]\nstatic_files = ["[z-a].txt"]\n', - ); - yield* writeLocalFunction(tempRoot.current, "hello-world"); - - yield* functionsDeploy(baseFlags); - - expect(out.stderrText).toContain("WARN:"); - expect(stripControlSequences(out.stdoutText)).toContain("Deployed Functions on project"); - }).pipe(Effect.provide(layer), Effect.ensuring(removeTempRoot)); - }, - ); - - it.live("deploys a function whose import map is a commented deno.jsonc", () => { - const out = mockOutput({ format: "text" }); - const api = mockCommandPlatformApi({ - handler: (request) => { - if (request.method === "GET") { - return Effect.succeed(jsonResponse(request, 200, [])); - } - return Effect.succeed( - jsonResponse(request, 201, { - id: "function-id", - slug: "hello-world", - name: "hello-world", - status: "ACTIVE", - version: 2, - created_at: 1_687_423_025_152, - updated_at: 1_687_423_025_152, - verify_jwt: true, - import_map: true, - entrypoint_path: "functions/hello-world/index.ts", - import_map_path: "functions/hello-world/deno.jsonc", - }), - ); - }, - }); - const layer = Layer.mergeAll( - buildTestRuntime({ - out, - api, - cliSettings: mockCommandSettings({ workdir: tempRoot.current }), - runtimeInfo: mockRuntimeInfo({ cwd: tempRoot.current }), - }), - Layer.succeed(YesFlag, false), - Stdio.layerTest({ - args: Effect.succeed(["functions", "deploy", "hello-world", "--use-api"]), - }), - ); - - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - yield* writeCliConfig(tempRoot.current); - const functionDir = path.join(tempRoot.current, "supabase", "functions", "hello-world"); - yield* fs.makeDirectory(functionDir, { recursive: true }); - yield* fs.writeFileString( - path.join(functionDir, "index.ts"), - "Deno.serve(() => new Response())\n", - ); - // Only a `.jsonc` import map exists (no `deno.json`), so discovery must fall through to it. - yield* fs.writeFileString( - path.join(functionDir, "deno.jsonc"), - ["// top-level comment", "{", " // nested comment", ' "imports": {}', "}", ""].join("\n"), - ); - - yield* functionsDeploy(baseFlags); - - expect(stripControlSequences(out.stdoutText)).toContain("Deployed Functions on project"); - }).pipe(Effect.provide(layer), Effect.ensuring(removeTempRoot)); - }); - - it.live( - "fails with the pre-refactor raw-errno message for a dangling symlink inside an import-map scope target", - () => { - const out = mockOutput({ format: "text" }); - const api = mockCommandPlatformApi({ - handler: (request) => Effect.succeed(jsonResponse(request, 200, [])), - }); - const layer = Layer.mergeAll( - buildTestRuntime({ - out, - api, - cliSettings: mockCommandSettings({ workdir: tempRoot.current }), - runtimeInfo: mockRuntimeInfo({ cwd: tempRoot.current }), - }), - Layer.succeed(YesFlag, false), - Stdio.layerTest({ - args: Effect.succeed(["functions", "deploy", "hello-world", "--use-api"]), - }), - ); - - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - yield* writeLocalFunction(tempRoot.current, "hello-world"); - yield* writeCliConfig(tempRoot.current); - const functionDir = path.join(tempRoot.current, "supabase", "functions", "hello-world"); - yield* fs.writeFileString( - path.join(functionDir, "deno.json"), - '{"imports":{},"scopes":{"x/":{"y":"./vendor/"}}}', - ); - const vendorDir = path.join(functionDir, "vendor"); - yield* fs.makeDirectory(vendorDir, { recursive: true }); - const danglingLink = path.join(vendorDir, "dangling.ts"); - yield* fs.symlink(path.join(vendorDir, "does-not-exist.ts"), danglingLink); - - const error = yield* functionsDeploy(baseFlags).pipe(Effect.flip); - - if (!(error instanceof Error)) { - throw new Error(`unexpected error: ${String(error)}`); - } - // Matches the message `git show ec079c9b3:apps/cli/src/shared/functions/deploy.ts` - // produces for this exact fixture: Node's own raw ENOENT text from `stat`, not a - // reformatted `PlatformError` message. - expect(error.message).toBe(`ENOENT: no such file or directory, stat '${danglingLink}'`); - }).pipe(Effect.provide(layer), Effect.ensuring(removeTempRoot)); - }, - ); }); diff --git a/apps/cli/src/shared/functions/deploy.errors.ts b/apps/cli/src/shared/functions/deploy.errors.ts index 97e234eb09..6b9198ca21 100644 --- a/apps/cli/src/shared/functions/deploy.errors.ts +++ b/apps/cli/src/shared/functions/deploy.errors.ts @@ -1,132 +1,52 @@ -import { Schema } from "effect"; +import { Data } from "effect"; import { actionability, type CliErrorActionabilityDeclaration, ErrorActionabilityId, } from "../telemetry/error-actionability.ts"; -export class ConflictingFunctionDeployFlagsError extends Schema.TaggedError()( +export class ConflictingFunctionDeployFlagsError extends Data.TaggedError( "ConflictingFunctionDeployFlagsError", - { - message: Schema.String, - }, -) { +)<{ + readonly message: string; +}> { get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { return actionability.provideFlags; } } -export class FunctionDeployJobsRequiresApiError extends Schema.TaggedError()( - "FunctionDeployJobsRequiresApiError", - { - message: Schema.String, - }, -) { - get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { - return actionability.provideFlags; - } -} - -export class InvalidFunctionDeploySlugError extends Schema.TaggedError()( +export class InvalidFunctionDeploySlugError extends Data.TaggedError( "InvalidFunctionDeploySlugError", - { - message: Schema.String, - }, -) { +)<{ + readonly message: string; +}> { get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { return actionability.provideFlags; } } -export class NoFunctionsToDeployError extends Schema.TaggedError()( - "NoFunctionsToDeployError", - { - message: Schema.String, - }, -) { +export class NoFunctionsToDeployError extends Data.TaggedError("NoFunctionsToDeployError")<{ + readonly message: string; +}> { get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { return actionability.provideFlags; } } -export class FunctionDeployCancelledError extends Schema.TaggedError()( - "FunctionDeployCancelledError", - { - message: Schema.String, - }, -) { +export class FunctionDeployCancelledError extends Data.TaggedError("FunctionDeployCancelledError")<{ + readonly message: string; +}> { get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { return actionability.cancelled; } } -export class FunctionImportNotDirectoryError extends Schema.TaggedError()( +export class FunctionImportNotDirectoryError extends Data.TaggedError( "FunctionImportNotDirectoryError", - { - message: Schema.String, - }, -) { - get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { - return actionability.invalidConfig; - } -} - -/** An asset, import map, or import-map target resolved outside its allowed root. */ -export class FunctionAssetOutsideRootError extends Schema.TaggedError()( - "FunctionAssetOutsideRootError", - { - message: Schema.String, - }, -) { - get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { - return actionability.invalidConfig; - } -} - -/** A configured entrypoint, static file, or import map resolved to a directory. */ -export class FunctionAssetIsDirectoryError extends Schema.TaggedError()( - "FunctionAssetIsDirectoryError", - { - message: Schema.String, - }, -) { +)<{ + readonly message: string; +}> { get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { return actionability.invalidConfig; } } - -/** A `static_files` glob pattern matched no files on disk. */ -export class FunctionStaticPatternNoMatchError extends Schema.TaggedError()( - "FunctionStaticPatternNoMatchError", - { - message: Schema.String, - }, -) { - get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { - return actionability.invalidConfig; - } -} - -/** A `deno.json`/`deno.jsonc` or import map file failed to parse. */ -export class FunctionImportMapParseError extends Schema.TaggedError()( - "FunctionImportMapParseError", - { - message: Schema.String, - }, -) { - get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { - return actionability.invalidConfig; - } -} - -/** The Docker bundler container exited unsuccessfully. */ -export class FunctionBundleFailedError extends Schema.TaggedError()( - "FunctionBundleFailedError", - { - message: Schema.String, - }, -) { - get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { - return actionability.toolFailed; - } -} diff --git a/apps/cli/src/shared/functions/deploy.ts b/apps/cli/src/shared/functions/deploy.ts index b991b994e2..b86e82613d 100644 --- a/apps/cli/src/shared/functions/deploy.ts +++ b/apps/cli/src/shared/functions/deploy.ts @@ -1,6 +1,6 @@ import { brotliCompressSync, constants as zlibConstants } from "node:zlib"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- pure, synchronous path-string math (no I/O), the same convention as command-internal/path-containment.ts and docker-ids.ts. -import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { chmod, mkdir, mkdtemp, readFile, readdir, realpath, rm, stat } from "node:fs/promises"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { URL } from "node:url"; import { FunctionResponse_Output, @@ -12,13 +12,11 @@ import { inferFunctionsManifest, type ResolvedFunctionConfig as ManifestFunctionConfig, } from "@supabase/config/effect"; -import { Cause, Config, Duration, Effect, FileSystem, Option, Ref, Schedule, Schema } from "effect"; -import type { PlatformError } from "effect/PlatformError"; +import { Duration, Effect, Option, Schema } from "effect"; import * as HttpBody from "effect/unstable/http/HttpBody"; import * as HttpClientError from "effect/unstable/http/HttpClientError"; import { promptYesNo } from "../../command-internal/prompt-yes-no.ts"; import { bitbucketCloneDir } from "../../command-internal/bitbucket-pipeline.ts"; -import { isPathContainedInRoot } from "../../command-internal/path-containment.ts"; import { CONTEXT_CANCELED_MESSAGE } from "../output/errors.ts"; import { Output } from "../output/output.service.ts"; import { bold } from "../../command-internal/colors.ts"; @@ -39,14 +37,8 @@ import { import { slimImagesEnabled } from "../services/slim-images.ts"; import { ConflictingFunctionDeployFlagsError, - FunctionAssetIsDirectoryError, - FunctionAssetOutsideRootError, - FunctionBundleFailedError, FunctionDeployCancelledError, - FunctionDeployJobsRequiresApiError, - FunctionImportMapParseError, FunctionImportNotDirectoryError, - FunctionStaticPatternNoMatchError, InvalidFunctionDeploySlugError, NoFunctionsToDeployError, } from "./deploy.errors.ts"; @@ -56,9 +48,6 @@ import { ensureDockerNamedVolume, ensureDockerNetwork, isDockerRunning, - type NativeFailure, - nativeFailure, - nativePlatformFailure, resolveDockerNetworkMode, resolveEdgeRuntimeVersion, resolveFunctionsDockerImage, @@ -378,11 +367,16 @@ function toApiRelativePath(cwd: string, hostPath: string) { return toSlash(relativePath.length > 0 ? relativePath : basename(resolved)); } -// Call sites always pass already-`realPath`-resolved, absolute candidates and -// roots, so the shared primitive's "both arguments must already be -// canonicalized" precondition always holds here. +function isContainedPath(root: string, candidate: string) { + const relativePath = relative(resolve(root), resolve(candidate)); + return ( + relativePath === "" || + (!isAbsolute(relativePath) && relativePath !== ".." && !relativePath.startsWith(`..${sep}`)) + ); +} + function isContainedInAnyPath(roots: ReadonlyArray, candidate: string) { - return roots.some((root) => isPathContainedInRoot(root, candidate)); + return roots.some((root) => isContainedPath(root, candidate)); } /** @@ -396,31 +390,22 @@ function hasParentPathSegment(relativePath: string) { .some((segment) => segment === ".."); } -function fsErrorCode(error: PlatformError): string | undefined { - const cause = error.reason.cause; - return cause instanceof Error && "code" in cause ? String(cause.code) : undefined; +async function realpathIfExists(pathname: string) { + try { + return await realpath(resolve(pathname)); + } catch (error) { + // ENOTDIR (a path routed through a file) is as nonexistent as ENOENT here. + if ( + error instanceof Error && + "code" in error && + (error.code === "ENOENT" || error.code === "ENOTDIR") + ) { + return resolve(pathname); + } + throw error; + } } -/** Curried `nativePlatformFailure` for `Effect.mapError`, labeling the failing path once. */ -const nativeFsError = (path: string) => (error: PlatformError) => - nativePlatformFailure(error, path); - -const realpathIfExists = Effect.fn("functions.deploy.realpathIfExists")(function* ( - fs: FileSystem.FileSystem, - pathname: string, -) { - return yield* fs.realPath(resolve(pathname)).pipe( - Effect.catchTag("PlatformError", (error) => { - // ENOTDIR (a path routed through a file) is as nonexistent as ENOENT here. - const code = fsErrorCode(error); - if (error.reason._tag === "NotFound" || code === "ENOTDIR") { - return Effect.succeed(resolve(pathname)); - } - return Effect.fail(nativePlatformFailure(error, pathname)); - }), - ); -}); - const resolveFunctionsSourceRoot = (projectRoot: string) => findGitRootPath(projectRoot).pipe(Effect.map(Option.getOrElse(() => resolve(projectRoot)))); @@ -555,27 +540,23 @@ function getObjectProperty(input: object, key: string): unknown { return Reflect.get(input, key); } -const readStringMap = Effect.fnUntraced(function* (input: unknown, fieldName: string) { +function readStringMap(input: unknown, fieldName: string): Record { if (input === undefined) { return {}; } if (typeof input !== "object" || input === null || Array.isArray(input)) { - return yield* new FunctionImportMapParseError({ - message: `failed to parse import map: expected ${fieldName} to be an object`, - }); + throw new Error(`failed to parse import map: expected ${fieldName} to be an object`); } const values: Record = {}; for (const [key, value] of Object.entries(input)) { if (typeof value !== "string") { - return yield* new FunctionImportMapParseError({ - message: `failed to parse import map: expected ${fieldName}.${key} to be a string`, - }); + throw new Error(`failed to parse import map: expected ${fieldName}.${key} to be a string`); } values[key] = value; } return values; -}); +} class ImportMapFile { readonly imports: Record; @@ -592,7 +573,7 @@ class ImportMapFile { this.importMapReference = importMapReference; } - static fromUnknown = Effect.fnUntraced(function* (input: unknown) { + static fromUnknown(input: unknown) { const imports: Record = {}; const scopes: Record> = {}; let importMapReference = ""; @@ -603,24 +584,22 @@ class ImportMapFile { importMapReference = importMap; } - Object.assign(imports, yield* readStringMap(getObjectProperty(input, "imports"), "imports")); + Object.assign(imports, readStringMap(getObjectProperty(input, "imports"), "imports")); const rawScopes = getObjectProperty(input, "scopes"); if (rawScopes === undefined) { return new ImportMapFile(imports, scopes, importMapReference); } if (typeof rawScopes !== "object" || rawScopes === null || Array.isArray(rawScopes)) { - return yield* new FunctionImportMapParseError({ - message: "failed to parse import map: expected scopes to be an object", - }); + throw new Error("failed to parse import map: expected scopes to be an object"); } for (const [scopeName, scopeValue] of Object.entries(rawScopes)) { - scopes[scopeName] = yield* readStringMap(scopeValue, `scopes.${scopeName}`); + scopes[scopeName] = readStringMap(scopeValue, `scopes.${scopeName}`); } } return new ImportMapFile(imports, scopes, importMapReference); - }); + } isReference() { return ( @@ -652,59 +631,28 @@ class ImportMapFile { } } -/** - * Shared by every JSONC import-map read (`loadImportMapFile`, `resolveImportMapAllowedRoots`). - * Stays on raw `JSON.parse`, not `Schema.fromJsonString`: that combinator's own JSON-parse step - * (`SchemaGetter.parseJson`) discards the native `SyntaxError` and reports a fixed - * "a valid JSON string" message instead, which would change this error's existing text for a - * malformed file. - */ -function decodeJsoncBytes( - contents: Uint8Array, -): Effect.Effect { - return Effect.try({ - // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- see this function's own doc comment: Schema.fromJsonString discards the native SyntaxError this error's message needs - try: () => JSON.parse(stripJsonComments(new TextDecoder().decode(contents))), - catch: (cause) => - new FunctionImportMapParseError({ - message: cause instanceof Error ? cause.message : String(cause), - }), - }); -} - -// Explicit type annotation breaks the circular inference from the recursive -// `yield* loadImportMapFile(...)` call below. -const loadImportMapFile: ( - fs: FileSystem.FileSystem, +async function loadImportMapFile( pathname: string, - onRead?: (pathname: string, contents: Uint8Array) => Effect.Effect, - seen?: Set, -) => Effect.Effect = - Effect.fnUntraced(function* ( - fs: FileSystem.FileSystem, - pathname: string, - onRead?: (pathname: string, contents: Uint8Array) => Effect.Effect, - seen: Set = new Set(), - ) { - const resolvedPath = resolve(pathname); - if (seen.has(resolvedPath)) { - return yield* new FunctionImportMapParseError({ - message: `cyclic import map reference: ${pathname}`, - }); - } - seen.add(resolvedPath); - const contents = yield* fs.readFile(pathname).pipe(Effect.mapError(nativeFsError(pathname))); - if (onRead !== undefined) { - yield* onRead(pathname, contents); - } - const parsed = yield* decodeJsoncBytes(contents); - const importMap = (yield* ImportMapFile.fromUnknown(parsed)).resolve(toSlash(pathname)); - if (isDenoConfigFile(pathname) && importMap.isReference()) { - const nestedPath = join(dirname(pathname), importMap.importMapReference); - return yield* loadImportMapFile(fs, nestedPath, onRead, seen); - } - return importMap; - }); + onRead?: (pathname: string, contents: Uint8Array) => Promise, + seen = new Set(), +): Promise { + const resolvedPath = resolve(pathname); + if (seen.has(resolvedPath)) { + throw new Error(`cyclic import map reference: ${pathname}`); + } + seen.add(resolvedPath); + const contents = await readFile(pathname); + if (onRead !== undefined) { + await onRead(pathname, contents); + } + const parsed = JSON.parse(stripJsonComments(new TextDecoder().decode(contents))); + const importMap = ImportMapFile.fromUnknown(parsed).resolve(toSlash(pathname)); + if (isDenoConfigFile(pathname) && importMap.isReference()) { + const nestedPath = join(dirname(pathname), importMap.importMapReference); + return loadImportMapFile(nestedPath, onRead, seen); + } + return importMap; +} function substituteImportMapValue( mappings: Readonly>, @@ -778,14 +726,13 @@ function resolveImportSpecifier( return { path: resolved, substituted }; } -const walkImportPaths = Effect.fnUntraced(function* ( - fs: FileSystem.FileSystem, +async function walkImportPaths( importMap: ImportMapFile, srcPath: string, allowedRoots: ReadonlyArray, displayRoot: string, - onFile: (pathname: string, contents: Uint8Array) => Effect.Effect, - onWarning: (message: string) => Effect.Effect, + onFile: (pathname: string, contents: Uint8Array) => Promise, + onWarning: (message: string) => Promise, ) { const seen = new Set(); const queue = [toSlash(srcPath)]; @@ -797,39 +744,33 @@ const walkImportPaths = Effect.fnUntraced(function* ( } seen.add(current); - const maybeContents: Option.Option = yield* Effect.gen(function* () { - const resolvedCurrent = yield* fs.realPath(resolve(current)); + let contents: Uint8Array; + try { + const resolvedCurrent = await realpath(resolve(current)); if (!isContainedInAnyPath(allowedRoots, resolvedCurrent)) { - yield* onWarning(`WARN: Skipping import path outside source root: ${current}\n`); - return Option.none(); + await onWarning(`WARN: Skipping import path outside source root: ${current}\n`); + continue; } - return Option.some(yield* fs.readFile(resolvedCurrent)); - }).pipe( - Effect.catchTag("PlatformError", (error) => - Effect.gen(function* () { - if (error.reason._tag === "NotFound") { - const message = `failed to read file: open ${toApiRelativePath(displayRoot, current)}: no such file or directory`; - yield* onWarning(`WARN: ${message}\n`); - return Option.none(); - } - // An ENOTDIR (import path routed through a file) gets a classified, user-facing message - // instead of an unhandled raw Node error, so telemetry books it as user-fixable config. - if (fsErrorCode(error) === "ENOTDIR") { - return yield* new FunctionImportNotDirectoryError({ - message: `failed to read file: open ${toApiRelativePath(displayRoot, current)}: not a directory`, - }); - } - return yield* nativePlatformFailure(error, current); - }), - ), - ); - - if (Option.isNone(maybeContents)) { - continue; + contents = await readFile(resolvedCurrent); + } catch (error) { + if (error instanceof Error && "code" in error) { + if (error.code === "ENOENT") { + const message = `failed to read file: open ${toApiRelativePath(displayRoot, current)}: no such file or directory`; + await onWarning(`WARN: ${message}\n`); + continue; + } + // An ENOTDIR (import path routed through a file) gets a classified, user-facing message + // instead of an unhandled raw Node error, so telemetry books it as user-fixable config. + if (error.code === "ENOTDIR") { + throw new FunctionImportNotDirectoryError({ + message: `failed to read file: open ${toApiRelativePath(displayRoot, current)}: not a directory`, + }); + } + } + throw error; } - const contents = maybeContents.value; - yield* onFile(current, contents); + await onFile(current, contents); const text = new TextDecoder().decode(contents); importPathPattern.lastIndex = 0; for (const match of text.matchAll(importPathPattern)) { @@ -867,15 +808,15 @@ const walkImportPaths = Effect.fnUntraced(function* ( } const resolvedModule = resolve(modulePath); - const containmentPath = yield* realpathIfExists(fs, resolvedModule); + const containmentPath = await realpathIfExists(resolvedModule); if (!isContainedInAnyPath(allowedRoots, containmentPath)) { - yield* onWarning(`WARN: Skipping import path outside source root: ${modulePath}\n`); + await onWarning(`WARN: Skipping import path outside source root: ${modulePath}\n`); continue; } queue.push(toSlash(resolvedModule)); } } -}); +} function hasGlobMeta(pattern: string) { return pattern.includes("*") || pattern.includes("?") || pattern.includes("["); @@ -944,161 +885,131 @@ function globBaseDirectory(pattern: string) { return stableParts.join("/"); } -// Node's native recursive `readdir` yields every intermediate directory alongside leaf files, -// relative to `root`. -const listPathsRecursive = Effect.fnUntraced(function* (fs: FileSystem.FileSystem, root: string) { +async function listPathsRecursive(root: string): Promise> { const resolvedRoot = resolve(root); - const entries = yield* fs.readDirectory(resolvedRoot, { recursive: true }); - return entries.map((entry) => join(resolvedRoot, entry)); -}); - -const expandStaticPattern = Effect.fn("functions.deploy.expandStaticPattern")(function* ( - fs: FileSystem.FileSystem, - pattern: string, -) { - const noMatch = () => - new FunctionStaticPatternNoMatchError({ message: `no files matched pattern: ${pattern}` }); + const entries = await readdir(resolvedRoot, { withFileTypes: true }); + const paths: string[] = []; + for (const entry of entries) { + const pathname = join(resolvedRoot, entry.name); + paths.push(pathname); + if (entry.isDirectory()) { + paths.push(...(await listPathsRecursive(pathname))); + } + } + return paths; +} +async function expandStaticPattern(pattern: string): Promise> { if (!hasGlobMeta(pattern)) { - const exists = yield* fs.stat(pattern).pipe( - Effect.as(true), - Effect.catchTag("PlatformError", () => Effect.succeed(false)), - ); - if (!exists) { - return yield* noMatch(); + try { + await stat(pattern); + } catch { + throw new Error(`no files matched pattern: ${pattern}`); } return [pattern]; } const baseDir = globBaseDirectory(pattern); - // `globToRegExp` builds its result with `new RegExp(...)`, which throws a native `SyntaxError` - // for an invalid bracket expression (e.g. `[z-a]`); keep that as a typed failure the recovery - // below can catch, not a defect that would bypass it. - const matcher = yield* Effect.try({ - try: () => globToRegExp(toSlash(resolve(pattern))), - catch: (cause) => nativeFailure(cause), - }); - const candidates = yield* listPathsRecursive(fs, baseDir).pipe( - Effect.catchTag( - "PlatformError", - (error): Effect.Effect => - error.reason._tag === "NotFound" - ? Effect.fail(noMatch()) - : Effect.fail(nativePlatformFailure(error, baseDir)), - ), - ); + const matcher = globToRegExp(toSlash(resolve(pattern))); + let candidates: ReadonlyArray; + try { + candidates = await listPathsRecursive(baseDir); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + throw new Error(`no files matched pattern: ${pattern}`); + } + throw error; + } const matches = candidates.filter((candidate) => matcher.test(toSlash(resolve(candidate)))); if (matches.length === 0) { - return yield* noMatch(); + throw new Error(`no files matched pattern: ${pattern}`); } return matches; -}); +} -const forEachLocalImportMapTarget = Effect.fnUntraced(function* ( +async function forEachLocalImportMapTarget( importMap: ImportMapFile, - onTarget: (pathname: string, kind: "import" | "scope") => Effect.Effect, + onTarget: (pathname: string, kind: "import" | "scope") => Promise, ) { for (const target of Object.values(importMap.imports)) { if (isRemoteImportTarget(target)) { continue; } - yield* onTarget(target, "import"); + await onTarget(target, "import"); } for (const scope of Object.values(importMap.scopes)) { for (const target of Object.values(scope)) { if (isRemoteImportTarget(target)) { continue; } - yield* onTarget(target, "scope"); + await onTarget(target, "scope"); } } -}); +} -const walkLocalImportMapTargetImports = Effect.fnUntraced(function* ( - fs: FileSystem.FileSystem, +async function walkLocalImportMapTargetImports( importMap: ImportMapFile, pathname: string, allowedRoots: ReadonlyArray, displayRoot: string, - onFile: (pathname: string, contents: Uint8Array) => Effect.Effect, - onWarning: (message: string) => Effect.Effect, + onFile: (pathname: string, contents: Uint8Array) => Promise, + onWarning: (message: string) => Promise, ) { - const info = yield* fs.stat(pathname).pipe(Effect.mapError(nativeFsError(pathname))); - if (info.type === "Directory") { + if ((await stat(pathname)).isDirectory()) { return; } - yield* walkImportPaths(fs, importMap, pathname, allowedRoots, displayRoot, onFile, onWarning); -}); + await walkImportPaths(importMap, pathname, allowedRoots, displayRoot, onFile, onWarning); +} -const isFile = Effect.fnUntraced(function* (fs: FileSystem.FileSystem, pathname: string) { - return yield* fs.stat(pathname).pipe( - Effect.map((info) => info.type === "File"), - Effect.catchTag("PlatformError", () => Effect.succeed(false)), - ); -}); +async function isFile(pathname: string): Promise { + try { + return (await stat(pathname)).isFile(); + } catch { + return false; + } +} -const resolveImportMapAllowedRoots = Effect.fnUntraced(function* ( - fs: FileSystem.FileSystem, - projectRoot: string, - importMapPath: string, -) { - const realProjectRoot = yield* fs - .realPath(projectRoot) - .pipe(Effect.mapError(nativeFsError(projectRoot))); +async function resolveImportMapAllowedRoots(projectRoot: string, importMapPath: string) { + const realProjectRoot = await realpath(projectRoot); const allowedRoots = [realProjectRoot]; if (importMapPath.length === 0) { return allowedRoots; } - const realImportMapPath = yield* fs - .realPath(importMapPath) - .pipe(Effect.mapError(nativeFsError(importMapPath))); - if (!isPathContainedInRoot(realProjectRoot, realImportMapPath)) { + const realImportMapPath = await realpath(importMapPath); + if (!isContainedPath(realProjectRoot, realImportMapPath)) { allowedRoots.push(dirname(realImportMapPath)); } if (isDenoConfigFile(importMapPath)) { - const contents = yield* fs - .readFile(importMapPath) - .pipe(Effect.mapError(nativeFsError(importMapPath))); - const parsed = yield* decodeJsoncBytes(contents); - const importMap = yield* ImportMapFile.fromUnknown(parsed); + const contents = await readFile(importMapPath); + const parsed = JSON.parse(stripJsonComments(new TextDecoder().decode(contents))); + const importMap = ImportMapFile.fromUnknown(parsed); if (importMap.importMapReference.length > 0) { - const referencedPath = join(dirname(importMapPath), importMap.importMapReference); - const referencedImportMapPath = yield* fs - .realPath(referencedPath) - .pipe(Effect.mapError(nativeFsError(referencedPath))); - if (!isPathContainedInRoot(realProjectRoot, referencedImportMapPath)) { + const referencedImportMapPath = await realpath( + join(dirname(importMapPath), importMap.importMapReference), + ); + if (!isContainedPath(realProjectRoot, referencedImportMapPath)) { allowedRoots.push(dirname(referencedImportMapPath)); } } } return allowedRoots; -}); +} -const writeSourceDeployForm = Effect.fn("functions.deploy.writeSourceDeployForm")(function* ( - fs: FileSystem.FileSystem, +async function writeSourceDeployForm( sourceRoot: string, workdir: string, config: ResolvedDeployFunctionConfig, + metadata: SourceDeployMetadata, outputRaw: (text: string) => Effect.Effect, ) { - const files: Array = []; - const hasImportMap = config.importMap.length > 0; - const realSourceRoot = yield* fs - .realPath(sourceRoot) - .pipe(Effect.mapError(nativeFsError(sourceRoot))); - const importMapAllowedRoots = yield* resolveImportMapAllowedRoots( - fs, - sourceRoot, - config.importMap, - ); + const form = new FormData(); + form.append("metadata", JSON.stringify(metadata)); + const realSourceRoot = await realpath(sourceRoot); + const importMapAllowedRoots = await resolveImportMapAllowedRoots(sourceRoot, config.importMap); const uploadedAssets = new Set(); - const appendAsset = Effect.fnUntraced(function* ( - pathname: string, - contents: Uint8Array, - realPathname: string, - ) { + const appendAsset = async (pathname: string, contents: Uint8Array, realPathname: string) => { if (uploadedAssets.has(realPathname)) { return; } @@ -1107,164 +1018,130 @@ const writeSourceDeployForm = Effect.fn("functions.deploy.writeSourceDeployForm" // `deployViaApi`. const relativePath = toApiRelativePath(workdir, pathname); if (hasParentPathSegment(relativePath)) { - return yield* new FunctionAssetOutsideRootError({ - message: `failed to read file: open ${relativePath}: invalid argument`, - }); + throw new Error(`failed to read file: open ${relativePath}: invalid argument`); } - yield* outputRaw(`Uploading asset (${config.slug}): ${relativePath}\n`); - files.push(new File([contents], relativePath)); - }); + await Effect.runPromise(outputRaw(`Uploading asset (${config.slug}): ${relativePath}\n`)); + form.append("file", new File([contents], relativePath)); + }; - const uploadAsset = Effect.fnUntraced(function* (pathname: string, contents: Uint8Array) { - const realPathname = yield* fs - .realPath(pathname) - .pipe(Effect.mapError(nativeFsError(pathname))); - if (!isPathContainedInRoot(realSourceRoot, realPathname)) { - return yield* new FunctionAssetOutsideRootError({ - message: `refusing to upload asset outside source root: ${pathname}`, - }); + const uploadAsset = async (pathname: string, contents: Uint8Array) => { + const realPathname = await realpath(pathname); + if (!isContainedPath(realSourceRoot, realPathname)) { + throw new Error(`refusing to upload asset outside source root: ${pathname}`); } - yield* appendAsset(pathname, contents, realPathname); - }); + await appendAsset(pathname, contents, realPathname); + }; - const uploadImportMapAsset = Effect.fnUntraced(function* ( - pathname: string, - contents: Uint8Array, - ) { - const realPathname = yield* fs - .realPath(pathname) - .pipe(Effect.mapError(nativeFsError(pathname))); + const uploadImportMapAsset = async (pathname: string, contents: Uint8Array) => { + const realPathname = await realpath(pathname); if (!isContainedInAnyPath(importMapAllowedRoots, realPathname)) { - return yield* new FunctionAssetOutsideRootError({ - message: `refusing to upload import map outside allowed roots: ${pathname}`, - }); + throw new Error(`refusing to upload import map outside allowed roots: ${pathname}`); } - yield* appendAsset(pathname, contents, realPathname); - }); + await appendAsset(pathname, contents, realPathname); + }; - const uploadImportMapTargetAsset = Effect.fnUntraced(function* ( - pathname: string, - contents: Uint8Array, - ) { - const realPathname = yield* fs - .realPath(pathname) - .pipe(Effect.mapError(nativeFsError(pathname))); + const uploadImportMapTargetAsset = async (pathname: string, contents: Uint8Array) => { + const realPathname = await realpath(pathname); if (!isContainedInAnyPath(importMapAllowedRoots, realPathname)) { - yield* outputRaw(`WARN: Skipping import path outside source root: ${pathname}\n`); + await Effect.runPromise( + outputRaw(`WARN: Skipping import path outside source root: ${pathname}\n`), + ); return; } - yield* appendAsset(pathname, contents, realPathname); - }); + await appendAsset(pathname, contents, realPathname); + }; - const uploadScopeTarget = Effect.fnUntraced(function* (pathname: string) { - const outcome = yield* fs.realPath(pathname).pipe( - Effect.flatMap((resolvedPath) => - fs.stat(pathname).pipe(Effect.map((pathInfo) => Option.some({ resolvedPath, pathInfo }))), - ), - Effect.catchTag("PlatformError", (error) => - fsErrorCode(error) === "ENOTDIR" - ? Effect.succeed( - Option.none<{ - readonly resolvedPath: string; - readonly pathInfo: FileSystem.File.Info; - }>(), - ) - : Effect.fail(nativePlatformFailure(error, pathname)), - ), - ); - if (Option.isNone(outcome)) { - yield* outputRaw(`WARN: Skipping import map target that is not a directory: ${pathname}\n`); - return; + const uploadScopeTarget = async (pathname: string) => { + let resolvedPath: string; + let pathInfo: Awaited>; + try { + resolvedPath = await realpath(pathname); + pathInfo = await stat(pathname); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOTDIR") { + await Effect.runPromise( + outputRaw(`WARN: Skipping import map target that is not a directory: ${pathname}\n`), + ); + return; + } + throw error; } - const { resolvedPath, pathInfo } = outcome.value; if (!isContainedInAnyPath(importMapAllowedRoots, resolvedPath)) { - yield* outputRaw(`WARN: Skipping import path outside source root: ${pathname}\n`); + await Effect.runPromise( + outputRaw(`WARN: Skipping import path outside source root: ${pathname}\n`), + ); return; } - if (pathInfo.type !== "Directory") { - const contents = yield* fs.readFile(pathname).pipe(Effect.mapError(nativeFsError(pathname))); - yield* uploadImportMapTargetAsset(pathname, contents); - yield* walkLocalImportMapTargetImports( - fs, + if (!pathInfo.isDirectory()) { + await uploadImportMapTargetAsset(pathname, await readFile(pathname)); + await walkLocalImportMapTargetImports( importMap, pathname, importMapAllowedRoots, workdir, uploadImportMapTargetAsset, - outputRaw, + async (message) => { + await Effect.runPromise(outputRaw(message)); + }, ); return; } - const nestedPaths = yield* listPathsRecursive(fs, pathname).pipe( - Effect.mapError(nativeFsError(pathname)), - ); + const nestedPaths = await listPathsRecursive(pathname); for (const nestedPath of nestedPaths) { - const nestedInfo = yield* fs - .stat(nestedPath) - .pipe(Effect.mapError(nativeFsError(nestedPath))); - if (nestedInfo.type === "Directory") { + if ((await stat(nestedPath)).isDirectory()) { continue; } - const resolvedNestedPath = yield* fs - .realPath(nestedPath) - .pipe(Effect.mapError(nativeFsError(nestedPath))); + const resolvedNestedPath = await realpath(nestedPath); if (!isContainedInAnyPath(importMapAllowedRoots, resolvedNestedPath)) { - yield* outputRaw(`WARN: Skipping import path outside source root: ${nestedPath}\n`); + await Effect.runPromise( + outputRaw(`WARN: Skipping import path outside source root: ${nestedPath}\n`), + ); continue; } - const nestedContents = yield* fs - .readFile(nestedPath) - .pipe(Effect.mapError(nativeFsError(nestedPath))); - yield* uploadImportMapTargetAsset(nestedPath, nestedContents); + await uploadImportMapTargetAsset(nestedPath, await readFile(nestedPath)); } - }); + }; - if (hasImportMap) { - yield* loadImportMapFile(fs, config.importMap, uploadImportMapAsset); + if (metadata.import_map_path !== undefined && metadata.import_map_path.length > 0) { + await loadImportMapFile(config.importMap, uploadImportMapAsset); } for (const pattern of config.staticFiles) { - // Every typed expansion failure (no match, an unreadable directory, ...) is warned and - // skipped. Interruptions and defects are untouched — only the two declared tags are caught. - const warnAndSkip = (error: { readonly message: string }) => - outputRaw(`WARN: ${error.message}\n`).pipe(Effect.as(undefined)); - const files: ReadonlyArray | undefined = yield* expandStaticPattern(fs, pattern).pipe( - Effect.catchTags({ - FunctionStaticPatternNoMatchError: warnAndSkip, - NativeFailure: warnAndSkip, - }), - ); - if (files === undefined) { + let files: ReadonlyArray; + try { + files = await expandStaticPattern(pattern); + } catch (error) { + await Effect.runPromise( + outputRaw(`WARN: ${error instanceof Error ? error.message : String(error)}\n`), + ); continue; } for (const pathname of files) { - const info = yield* fs.stat(pathname).pipe(Effect.mapError(nativeFsError(pathname))); - if (info.type === "Directory") { - return yield* new FunctionAssetIsDirectoryError({ - message: `file path is a directory: ${pathname}`, - }); + if ((await stat(pathname)).isDirectory()) { + throw new Error(`file path is a directory: ${pathname}`); } - const contents = yield* fs.readFile(pathname).pipe(Effect.mapError(nativeFsError(pathname))); - yield* uploadAsset(pathname, contents); + await uploadAsset(pathname, await readFile(pathname)); } } - const importMap = hasImportMap - ? yield* loadImportMapFile(fs, config.importMap) - : new ImportMapFile(); - yield* walkImportPaths( - fs, + const importMap = + metadata.import_map_path !== undefined && metadata.import_map_path.length > 0 + ? await loadImportMapFile(config.importMap) + : new ImportMapFile(); + await walkImportPaths( importMap, config.entrypoint, [realSourceRoot], workdir, uploadAsset, - outputRaw, + async (message) => { + await Effect.runPromise(outputRaw(message)); + }, ); - yield* forEachLocalImportMapTarget(importMap, uploadScopeTarget); + await forEachLocalImportMapTarget(importMap, uploadScopeTarget); - return files; -}); + return form; +} /** * Server-recorded metadata paths are anchored at the workdir, with forward slashes regardless of @@ -1332,7 +1209,7 @@ function sanitizeDockerBinds( type DockerBindsOptions = { readonly additionalModuleRoots?: ReadonlyArray; - readonly onWarning?: (message: string) => Effect.Effect; + readonly onWarning?: (message: string) => Promise; readonly skipMissingImportMapTargets?: boolean; /** Resolved marker presence, including an explicitly empty project value. */ readonly bitbucketCloneDirDefined?: boolean; @@ -1347,40 +1224,38 @@ export const buildDockerBinds = ( ) => resolveFunctionsSourceRoot(resolve(functionsDir, "..", "..")).pipe( Effect.flatMap((sourceRoot) => - Effect.flatMap(FileSystem.FileSystem, (fs) => - buildDockerBindsWithin(fs, sourceRoot, projectId, functionsDir, outputDir, config, options), + Effect.promise(() => + buildDockerBindsWithin(sourceRoot, projectId, functionsDir, outputDir, config, options), ), ), ); -const buildDockerBindsWithin = Effect.fn("functions.deploy.buildDockerBinds")(function* ( - fs: FileSystem.FileSystem, +async function buildDockerBindsWithin( sourceRoot: string, projectId: string, functionsDir: string, outputDir: string, config: ResolvedDeployFunctionConfig, options: DockerBindsOptions, -) { +): Promise> { const hostFunctionsDir = resolve(functionsDir); const hostOutputDir = resolve(outputDir); - const realSourceRoot = yield* fs - .realPath(sourceRoot) - .pipe(Effect.mapError(nativeFsError(sourceRoot))); - const resolvedAdditionalRoots = yield* Effect.forEach( - options.additionalModuleRoots ?? [], - (root) => fs.realPath(root).pipe(Effect.option), - { concurrency: "unbounded" }, - ); + const realSourceRoot = await realpath(sourceRoot); const moduleRoots = [ realSourceRoot, - ...resolvedAdditionalRoots.flatMap((root) => (Option.isSome(root) ? [root.value] : [])), + ...( + await Promise.all( + (options.additionalModuleRoots ?? []).map(async (root) => { + try { + return await realpath(root); + } catch { + return undefined; + } + }), + ) + ).flatMap((root) => (root === undefined ? [] : [root])), ]; - const importMapAllowedRoots = yield* resolveImportMapAllowedRoots( - fs, - sourceRoot, - config.importMap, - ); + const importMapAllowedRoots = await resolveImportMapAllowedRoots(sourceRoot, config.importMap); const binds: DockerBind[] = [ { hostPath: hostFunctionsDir, @@ -1408,18 +1283,11 @@ const buildDockerBindsWithin = Effect.fn("functions.deploy.buildDockerBinds")(fu }); } - const onWarning = options.onWarning; - const warn = (message: string) => (onWarning === undefined ? Effect.void : onWarning(message)); + const warn = options.onWarning ?? (async () => {}); const extraBinds: DockerBind[] = []; const explicitScopeBinds = new Map(); - // Left as an untyped `PlatformError`, not wrapped into `NativeFailure`: the only caller that - // inspects a failure here — the scope-target walk below — classifies ENOTDIR/ENOENT off the - // platform tag. - const appendBindWithinRoots = Effect.fnUntraced(function* ( - roots: ReadonlyArray, - pathname: string, - ) { - const hostPath = yield* fs.realPath(pathname); + const appendBindWithinRoots = async (roots: ReadonlyArray, pathname: string) => { + const hostPath = await realpath(pathname); const contained = isContainedInAnyPath(roots, hostPath); if (contained) { extraBinds.push({ @@ -1430,19 +1298,21 @@ const buildDockerBindsWithin = Effect.fn("functions.deploy.buildDockerBinds")(fu }); } return { hostPath, contained }; - }); - const appendProjectBind = (pathname: string, _contents: Uint8Array) => - appendBindWithinRoots([realSourceRoot], pathname).pipe(Effect.asVoid); - const appendModuleBind = (pathname: string, _contents: Uint8Array) => - appendBindWithinRoots(moduleRoots, pathname).pipe(Effect.asVoid); - const appendImportMapBind = (pathname: string, _contents: Uint8Array) => - appendBindWithinRoots(importMapAllowedRoots, pathname).pipe(Effect.asVoid); + }; + const appendProjectBind = async (pathname: string, _contents: Uint8Array) => { + await appendBindWithinRoots([realSourceRoot], pathname); + }; + const appendModuleBind = async (pathname: string, _contents: Uint8Array) => { + await appendBindWithinRoots(moduleRoots, pathname); + }; + const appendImportMapBind = async (pathname: string, _contents: Uint8Array) => { + await appendBindWithinRoots(importMapAllowedRoots, pathname); + }; const importMap = config.importMap.length > 0 - ? yield* loadImportMapFile(fs, config.importMap, appendImportMapBind) + ? await loadImportMapFile(config.importMap, appendImportMapBind) : new ImportMapFile(); - yield* walkImportPaths( - fs, + await walkImportPaths( importMap, config.entrypoint, moduleRoots, @@ -1450,10 +1320,10 @@ const buildDockerBindsWithin = Effect.fn("functions.deploy.buildDockerBinds")(fu appendModuleBind, warn, ); - yield* forEachLocalImportMapTarget(importMap, (target, kind) => - Effect.gen(function* () { - const { hostPath, contained } = yield* appendBindWithinRoots(importMapAllowedRoots, target); - const isDirectory = (yield* fs.stat(target)).type === "Directory"; + await forEachLocalImportMapTarget(importMap, async (target, kind) => { + try { + const { hostPath, contained } = await appendBindWithinRoots(importMapAllowedRoots, target); + const isDirectory = (await stat(target)).isDirectory(); if (!contained && kind === "scope") { const scopeBind: DockerBind = { hostPath, @@ -1466,46 +1336,44 @@ const buildDockerBindsWithin = Effect.fn("functions.deploy.buildDockerBinds")(fu if (isDirectory) { return; } - yield* walkLocalImportMapTargetImports( - fs, + await walkLocalImportMapTargetImports( importMap, target, importMapAllowedRoots, sourceRoot, appendImportMapBind, - () => Effect.void, + async () => {}, ); - }).pipe( - Effect.catchTag("PlatformError", (error) => { + } catch (error) { + if (error instanceof Error && "code" in error) { // ENOTDIR (a trailing-slash value routed through a file) is never a // walkable target regardless of caller: an import that actually // reaches through that file still fails via the walker's // FunctionImportNotDirectoryError. - if (fsErrorCode(error) === "ENOTDIR") { - return warn(`WARN: Skipping import map target that is not a directory: ${target}\n`); + if (error.code === "ENOTDIR") { + await warn(`WARN: Skipping import map target that is not a directory: ${target}\n`); + return; } - if (options.skipMissingImportMapTargets === true && error.reason._tag === "NotFound") { - return warn(`WARN: Skipping missing import map target: ${target}\n`); + if (options.skipMissingImportMapTargets === true && error.code === "ENOENT") { + await warn(`WARN: Skipping missing import map target: ${target}\n`); + return; } - return Effect.fail(nativePlatformFailure(error, target)); - }), - ), - ); + } + throw error; + } + }); for (const pattern of config.staticFiles) { - // Any expansion failure (no match, unreadable directory, ...) is silently skipped here, - // unlike the API-upload path's warned skip. - const files = yield* expandStaticPattern(fs, pattern).pipe(Effect.option); - if (Option.isNone(files)) { + let files: ReadonlyArray; + try { + files = await expandStaticPattern(pattern); + } catch { continue; } - for (const pathname of files.value) { - const info = yield* fs.stat(pathname).pipe(Effect.mapError(nativeFsError(pathname))); - if (info.type === "Directory") { - return yield* new FunctionAssetIsDirectoryError({ - message: `file path is a directory: ${pathname}`, - }); + for (const pathname of files) { + if ((await stat(pathname)).isDirectory()) { + throw new Error(`file path is a directory: ${pathname}`); } - yield* appendProjectBind(pathname, new Uint8Array()); + await appendProjectBind(pathname, new Uint8Array()); } } @@ -1520,31 +1388,29 @@ const buildDockerBindsWithin = Effect.fn("functions.deploy.buildDockerBinds")(fu } occupiedContainerPaths.add(bind.containerPath); uniqueScopeBinds.push(bind); - yield* warn( + await warn( `WARN: Mounting import map scope target outside the project root: ${bind.hostPath}\n`, ); } return [...binds, ...sanitizedExtraBinds, ...uniqueScopeBinds]; -}); +} function shouldUseDenoJsonDiscovery(entrypoint: string, importMap: string) { return isDenoConfigFile(importMap) && dirname(importMap) === dirname(entrypoint); } -const shouldUsePackageJsonDiscovery = Effect.fnUntraced(function* ( - fs: FileSystem.FileSystem, - entrypoint: string, - importMap: string, -) { +async function shouldUsePackageJsonDiscovery(entrypoint: string, importMap: string) { if (importMap.length > 0) { return false; } - return yield* fs.stat(join(dirname(entrypoint), "package.json")).pipe( - Effect.as(true), - Effect.catchTag("PlatformError", () => Effect.succeed(false)), - ); -}); + try { + await stat(join(dirname(entrypoint), "package.json")); + return true; + } catch { + return false; + } +} interface BundleFunctionWithDockerOptions { readonly projectId: string; @@ -1573,194 +1439,184 @@ const bundleFunctionWithDocker = Effect.fn("functions.deploy.bundleWithDocker")( } = options; const bitbucketCloneDirDefined = Option.isSome(yield* bitbucketCloneDir(projectEnvValues)); const output = yield* Output; - const fs = yield* FileSystem.FileSystem; yield* output.raw(`Bundling Function: ${styleEmphasis(config.slug)}\n`, "stderr"); const outputRoot = resolve(functionsDir, "..", ".temp"); - yield* fs - .makeDirectory(outputRoot, { recursive: true }) - .pipe(Effect.mapError(nativeFsError(outputRoot))); - // A generator `finally` does not run when a yielded effect fails or is interrupted. - return yield* Effect.acquireUseRelease( - fs - .makeTempDirectory({ directory: outputRoot, prefix: `.supabase-output-${config.slug}-` }) - .pipe(Effect.mapError(nativeFsError(outputRoot))), - (outputDir) => - Effect.gen(function* () { - // Go passes 0777 to MkdirAll, which Windows ignores. Calling chmod separately - // adds an NTFS WRITE_ATTRIBUTES requirement that the Go CLI does not have. - if (shouldChmodBundleOutputDirectory(process.platform)) { - yield* fs.chmod(outputDir, 0o777).pipe(Effect.mapError(nativeFsError(outputDir))); - } - const outputPath = join(outputDir, "output.eszip"); - // `edgeRuntimeImage` applies the tag verbatim — a `.temp/edge-runtime-version` pin flows - // through unmodified, `v` prefix or not (see the helper's doc in `functions.shared.ts`). - const rawImage = edgeRuntimeImage(edgeRuntimeVersion, yield* slimImagesEnabled); - const binds = yield* buildDockerBinds(projectId, functionsDir, outputDir, config, { - bitbucketCloneDirDefined, - onWarning: (message) => output.raw(message, "stderr"), - }); - // Resolved per function rather than hoisted out of the loop (unlike `download.ts`'s - // `PulledEdgeRuntimeImage`): the first resolve failure aborts the loop, and the only added - // cost is one cached `docker image inspect` per function. - const image = yield* resolveFunctionsDockerImage(rawImage, projectEnvValues); - yield* ensureDockerNetwork(networkMode, projectId); - yield* ensureDockerNamedVolume( - edgeRuntimeCacheVolume(projectId).name, - projectId, - projectEnvValues, - ); + yield* Effect.tryPromise(() => mkdir(outputRoot, { recursive: true })); + const outputDir = yield* Effect.tryPromise(() => + mkdtemp(join(outputRoot, `.supabase-output-${config.slug}-`)), + ); + try { + // Go passes 0777 to MkdirAll, which Windows ignores. Calling chmod separately + // adds an NTFS WRITE_ATTRIBUTES requirement that the Go CLI does not have. + if (shouldChmodBundleOutputDirectory(process.platform)) { + yield* Effect.tryPromise({ + try: () => chmod(outputDir, 0o777), + catch: (cause) => (cause instanceof Error ? cause : new Error(String(cause))), + }); + } + const outputPath = join(outputDir, "output.eszip"); + // `edgeRuntimeImage` applies the tag verbatim — a `.temp/edge-runtime-version` pin flows + // through unmodified, `v` prefix or not (see the helper's doc in `functions.shared.ts`). + const rawImage = edgeRuntimeImage(edgeRuntimeVersion, yield* slimImagesEnabled); + const binds = yield* buildDockerBinds(projectId, functionsDir, outputDir, config, { + bitbucketCloneDirDefined, + onWarning: (message) => Effect.runPromise(output.raw(message, "stderr")), + }); + // Resolved per function rather than hoisted out of the loop (unlike `download.ts`'s + // `PulledEdgeRuntimeImage`): the first resolve failure aborts the loop, and the only added + // cost is one cached `docker image inspect` per function. + const image = yield* resolveFunctionsDockerImage(rawImage, projectEnvValues); + yield* ensureDockerNetwork(networkMode, projectId); + yield* ensureDockerNamedVolume( + edgeRuntimeCacheVolume(projectId).name, + projectId, + projectEnvValues, + ); - const env: Array = []; - if (!(yield* shouldUsePackageJsonDiscovery(fs, config.entrypoint, config.importMap))) { - env.push("DENO_NO_PACKAGE_JSON=1"); - } - env.push(...dockerNpmEnv()); - - const containerArgs = [ - "bundle", - "--entrypoint", - toDockerPath(config.entrypoint, { resolve }), - "--output", - toDockerPath(outputPath, { resolve }), - ]; - if ( - config.importMap.length > 0 && - !shouldUseDenoJsonDiscovery(config.entrypoint, config.importMap) - ) { - containerArgs.push("--import-map", toDockerPath(config.importMap, { resolve })); - } - for (const staticFile of config.staticFiles) { - containerArgs.push("--static", toDockerPath(staticFile, { resolve })); - } - const debugEnv = yield* Config.option(Config.string("DEBUG")); - if (verbose || Option.contains(debugEnv, "true")) { - containerArgs.push("--verbose"); - } + const env: Array = []; + if ( + !(yield* Effect.promise(() => + shouldUsePackageJsonDiscovery(config.entrypoint, config.importMap), + )) + ) { + env.push("DENO_NO_PACKAGE_JSON=1"); + } + env.push(...dockerNpmEnv()); + + const containerArgs = [ + "bundle", + "--entrypoint", + toDockerPath(config.entrypoint, { resolve }), + "--output", + toDockerPath(outputPath, { resolve }), + ]; + if ( + config.importMap.length > 0 && + !shouldUseDenoJsonDiscovery(config.entrypoint, config.importMap) + ) { + containerArgs.push("--import-map", toDockerPath(config.importMap, { resolve })); + } + for (const staticFile of config.staticFiles) { + containerArgs.push("--static", toDockerPath(staticFile, { resolve })); + } + if (verbose || process.env["DEBUG"] === "true") { + containerArgs.push("--verbose"); + } - const command = buildFunctionsDockerRunArgs({ - image, - projectId, - networkMode, - binds: binds.map(formatDockerBind), - env, - // `functionsDir` is `/supabase/functions`, same derivation as `deployViaApi`'s - // own `projectRoot`. - workingDir: toDockerPath(resolve(functionsDir, "..", ".."), { resolve }), - containerArgs, - }); + const command = buildFunctionsDockerRunArgs({ + image, + projectId, + networkMode, + binds: binds.map(formatDockerBind), + env, + // `functionsDir` is `/supabase/functions`, same derivation as `deployViaApi`'s + // own `projectRoot`. + workingDir: toDockerPath(resolve(functionsDir, "..", ".."), { resolve }), + containerArgs, + }); - // Live-tees each chunk to `output.raw` as it arrives, rather than buffering the whole run - // until exit. - const result = yield* runChildProcess("docker", command, { - stdout: "pipe", - stderr: "pipe", - onStdout: (chunk) => output.raw(chunk, output.format === "text" ? "stdout" : "stderr"), - onStderr: (chunk) => output.raw(chunk, "stderr"), - }); - if (result.exitCode !== 0) { - return yield* new FunctionBundleFailedError({ - message: `failed to bundle function: exit ${result.exitCode}`, - }); - } + // Live-tees each chunk to `output.raw` as it arrives, rather than buffering the whole run + // until exit. + const result = yield* runChildProcess("docker", command, { + stdout: "pipe", + stderr: "pipe", + onStdout: (chunk) => output.raw(chunk, output.format === "text" ? "stdout" : "stderr"), + onStderr: (chunk) => output.raw(chunk, "stderr"), + }); + if (result.exitCode !== 0) { + return yield* Effect.fail(new Error(`failed to bundle function: exit ${result.exitCode}`)); + } - const eszip = yield* fs.readFile(outputPath).pipe( - Effect.mapError( - (error) => - new FunctionBundleFailedError({ - message: `failed to open eszip: ${nativePlatformFailure(error, outputPath).message}`, - }), - ), - ); - const compressed = new Uint8Array( - Buffer.concat([ - Buffer.from(COMPRESSED_ESZIP_MAGIC), - brotliCompressSync(eszip, { - params: { - [zlibConstants.BROTLI_PARAM_QUALITY]: 6, - }, - }), - ]), - ); - const sha256 = yield* Effect.promise(() => crypto.subtle.digest("SHA-256", compressed)); - const hash = Buffer.from(sha256).toString("hex"); - yield* Effect.annotateCurrentSpan({ "bundle.bytes": compressed.byteLength }); - return { - slug: config.slug, - metadata: createBundledMetadata(config, hash), - body: compressed, - } satisfies BundledFunction; - }), - (outputDir) => - fs - .remove(outputDir, { recursive: true, force: true }) - .pipe(Effect.orElseSucceed(() => undefined)), - ); + const eszip = yield* Effect.tryPromise({ + try: () => readFile(outputPath), + catch: (error) => + new Error( + `failed to open eszip: ${error instanceof Error ? error.message : String(error)}`, + ), + }); + const compressed = new Uint8Array( + Buffer.concat([ + Buffer.from(COMPRESSED_ESZIP_MAGIC), + brotliCompressSync(eszip, { + params: { + [zlibConstants.BROTLI_PARAM_QUALITY]: 6, + }, + }), + ]), + ); + const sha256 = yield* Effect.promise(() => crypto.subtle.digest("SHA-256", compressed)); + const hash = Buffer.from(sha256).toString("hex"); + yield* Effect.annotateCurrentSpan({ "bundle.bytes": compressed.byteLength }); + return { + slug: config.slug, + metadata: createBundledMetadata(config, hash), + body: compressed, + } satisfies BundledFunction; + } finally { + yield* Effect.tryPromise(() => rm(outputDir, { recursive: true, force: true })).pipe( + Effect.orElseSucceed(() => undefined), + ); + } }); -// Transient-API bound shared by the list/bulk-update retry policies below: 3 retries (4 attempts -// total) with a 1s-base exponential backoff. -const transientApiRetrySchedule = Schedule.exponential("1 second").pipe( - Schedule.upTo({ times: 3 }), -); - -// Same 3-retry bound as `transientApiRetrySchedule`, with a 500ms base for `upsertBundledFunction`. -const upsertRetrySchedule = Schedule.exponential("500 millis").pipe(Schedule.upTo({ times: 3 })); - -/** Retries a transport-ish failure or a 5xx/429 status; a decode failure or other 4xx is terminal. */ -function isRetryableFunctionsApiError( - error: - | FunctionsApiStatusError - | FunctionsApiTransportError - | SupabaseApiInputError - | HttpBody.HttpBodyError, -): boolean { - return ( - !(error instanceof FunctionsApiStatusError) || - (error.decode !== true && (error.status >= 500 || error.status === 429)) - ); -} - const listRemoteFunctions = Effect.fn("functions.deploy.listRemoteFunctions")(function* ( api: ApiClient, projectRef: string, ) { - const attempt = Effect.gen(function* () { - const response = yield* api + let lastError: Error | FunctionsApiStatusError | undefined; + for (let attempt = 0; attempt <= 3; attempt += 1) { + const result = yield* api .executeRaw(operationDefinitions.v1ListAllFunctions, { ref: projectRef }) - .pipe(Effect.mapError((error) => mapTransportError("failed to list functions", error))); - const body = yield* response.text.pipe(Effect.orElseSucceed(() => "")); - if (response.status === 200) { - // A 200 whose body is not the expected JSON is an API-response problem, - // not a transport failure — surface it via FunctionsApiStatusError so it - // classifies as api_status rather than network. - return yield* Effect.try({ - // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- Schema.fromJsonString discards the native SyntaxError on malformed JSON (see decodeJsoncBytes); this raw JSON.parse keeps that message for the decode-error branch below - try: () => decodeFunctionListResponse(JSON.parse(body)), - catch: (error) => - new FunctionsApiStatusError({ - status: response.status, - message: `failed to read functions list: ${error instanceof Error ? error.message : String(error)}`, - decode: true, + .pipe( + Effect.map((response) => ({ success: true as const, response })), + Effect.catch((error) => + Effect.succeed({ + success: false as const, + error: mapTransportError("failed to list functions", error), }), + ), + ); + + if (result.success) { + const body = yield* result.response.text.pipe(Effect.orElseSucceed(() => "")); + if (result.response.status === 200) { + // A 200 whose body is not the expected JSON is an API-response problem, + // not a transport failure — surface it via FunctionsApiStatusError so it + // classifies as api_status rather than network. + return yield* Effect.try({ + try: () => decodeFunctionListResponse(JSON.parse(body)), + catch: (error) => + new FunctionsApiStatusError({ + status: result.response.status, + message: `failed to read functions list: ${error instanceof Error ? error.message : String(error)}`, + decode: true, + }), + }); + } + lastError = new FunctionsApiStatusError({ + status: result.response.status, + message: `unexpected list functions status ${result.response.status}: ${body}`, }); + if (result.response.status < 500 && result.response.status !== 429) { + return yield* Effect.fail(lastError); + } + } else { + lastError = result.error; } - return yield* new FunctionsApiStatusError({ - status: response.status, - message: `unexpected list functions status ${response.status}: ${body}`, - }); - }); - return yield* attempt.pipe( - Effect.retry({ schedule: transientApiRetrySchedule, while: isRetryableFunctionsApiError }), - ); + if (attempt < 3) { + yield* Effect.sleep(Duration.millis(1_000 * 2 ** attempt)); + } + } + return yield* Effect.fail(lastError ?? new Error("failed to list functions")); }); function headerValue(headers: Readonly>, name: string) { return headers[name.toLowerCase()] ?? headers[name]; } -function parseRateLimitDelay(value: string | undefined, nowMillis: number): number | undefined { +function parseRateLimitDelay(value: string | undefined): number | undefined { if (value === undefined || value.length === 0) { return undefined; } @@ -1770,20 +1626,18 @@ function parseRateLimitDelay(value: string | undefined, nowMillis: number): numb } const timestamp = Date.parse(value); if (!Number.isNaN(timestamp)) { - return Math.max(timestamp - nowMillis, 0); + return Math.max(timestamp - Date.now(), 0); } return undefined; } -/** `nowMillis` comes from the caller's `Schedule`/`Clock` metadata, never read from the system clock here. */ function rateLimitDelayMillis( headers: Readonly>, attempt: number, - nowMillis: number, ) { return ( - parseRateLimitDelay(headerValue(headers, "retry-after"), nowMillis) ?? - parseRateLimitDelay(headerValue(headers, "x-ratelimit-reset"), nowMillis) ?? + parseRateLimitDelay(headerValue(headers, "retry-after")) ?? + parseRateLimitDelay(headerValue(headers, "x-ratelimit-reset")) ?? 1_000 * 2 ** Math.min(attempt, 5) ); } @@ -1792,40 +1646,30 @@ function rateLimitDelayText(milliseconds: number) { return `${Math.round(milliseconds / 1_000)}s`; } -interface RateLimitableResponse { - readonly status: number; - readonly headers: Readonly>; - readonly body: Effect.Effect; -} - const rateLimitedRequest = Effect.fnUntraced(function* ( action: string, - request: () => Effect.Effect, Error>, + request: () => Effect.Effect< + { + readonly status: number; + readonly headers: Readonly>; + readonly body: Effect.Effect; + }, + Error + >, ) { const output = yield* Output; - // Repeats on a 429, reporting the exact header- or backoff-derived delay through `Schedule`'s - // own metadata. - const schedule = Schedule.fromStepWithMetadata( - Effect.succeed((meta: Schedule.InputMetadata>) => { - const previousAttempts = meta.attempt - 1; - if (meta.input.status !== 429 || previousAttempts >= DEPLOY_RATE_LIMIT_MAX_RETRIES) { - return Cause.done(meta.input); - } - const delayMs = rateLimitDelayMillis(meta.input.headers, previousAttempts, meta.now); - return output - .raw( - `Rate limit exceeded while ${action}. Retrying in ${rateLimitDelayText(delayMs)}.\n`, - "stderr", - ) - .pipe( - Effect.as([meta.input, Duration.millis(delayMs)] as [ - RateLimitableResponse, - Duration.Duration, - ]), - ); - }), - ); - return yield* request().pipe(Effect.repeat({ schedule })); + for (let attempt = 0; ; attempt += 1) { + const response = yield* request(); + if (response.status !== 429 || attempt >= DEPLOY_RATE_LIMIT_MAX_RETRIES) { + return response; + } + const delayMs = rateLimitDelayMillis(response.headers, attempt); + yield* output.raw( + `Rate limit exceeded while ${action}. Retrying in ${rateLimitDelayText(delayMs)}.\n`, + "stderr", + ); + yield* Effect.sleep(Duration.millis(delayMs)); + } }); const uploadFunctionSource = Effect.fn("functions.deploy.uploadFunctionSource")(function* ( @@ -1838,10 +1682,15 @@ const uploadFunctionSource = Effect.fn("functions.deploy.uploadFunctionSource")( bundleOnly: boolean, ) { const output = yield* Output; - const fs = yield* FileSystem.FileSystem; - const files = yield* writeSourceDeployForm(fs, sourceRoot, workdir, config, (text) => - output.raw(text, "stderr"), - ); + const files = yield* Effect.tryPromise({ + try: async () => { + const form = await writeSourceDeployForm(sourceRoot, workdir, config, metadata, (text) => + output.raw(text, "stderr"), + ); + return form.getAll("file").flatMap((part) => (part instanceof Blob ? [part] : [])); + }, + catch: (error) => (error instanceof Error ? error : new Error(String(error))), + }); const response = yield* rateLimitedRequest(`deploying function ${config.slug}`, () => api .executeRaw(operationDefinitions.v1DeployAFunction, { @@ -1868,16 +1717,17 @@ const uploadFunctionSource = Effect.fn("functions.deploy.uploadFunctionSource")( ); const body = yield* response.body; if (response.status !== 201) { - return yield* new FunctionsApiStatusError({ - status: response.status, - message: `unexpected deploy status ${response.status}: ${formatUnexpectedStatusBody(body)}`, - }); + return yield* Effect.fail( + new FunctionsApiStatusError({ + status: response.status, + message: `unexpected deploy status ${response.status}: ${formatUnexpectedStatusBody(body)}`, + }), + ); } // A 201 whose body is not the expected JSON is an API-response problem, not a // transport failure — surface it via FunctionsApiStatusError so it classifies // as api_status rather than network. return yield* Effect.try({ - // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- Schema.fromJsonString discards the native SyntaxError on malformed JSON (see decodeJsoncBytes); this raw JSON.parse keeps that message for the decode-error branch below try: () => decodeDeployFunctionResponse(JSON.parse(body)), catch: (error) => new FunctionsApiStatusError({ @@ -1909,8 +1759,9 @@ const bulkUpdateRemoteFunctions = Effect.fn("functions.deploy.bulkUpdateFunction projectRef: string, functions: ReadonlyArray, ) { - const attempt = Effect.gen(function* () { - const response = yield* rateLimitedRequest("bulk updating functions", () => + let lastError: Error | FunctionsApiStatusError | undefined; + for (let attempt = 0; attempt <= 3; attempt += 1) { + const result = yield* rateLimitedRequest("bulk updating functions", () => api .executeRaw(operationDefinitions.v1BulkUpdateFunctions, { ref: projectRef, @@ -1926,23 +1777,37 @@ const bulkUpdateRemoteFunctions = Effect.fn("functions.deploy.bulkUpdateFunction })), Effect.mapError((error) => mapTransportError("failed to bulk update", error)), ), + ).pipe( + Effect.map((response) => ({ success: true as const, response })), + Effect.catch((error) => + Effect.succeed({ + success: false as const, + error, + }), + ), ); - if (response.status === 200) { - return; + + if (result.success) { + const body = yield* result.response.body; + if (result.response.status === 200) { + return; + } + lastError = new FunctionsApiStatusError({ + status: result.response.status, + message: `unexpected bulk update status ${result.response.status}: ${body}`, + }); + if (result.response.status < 500) { + return yield* Effect.fail(lastError); + } + } else { + lastError = result.error; } - const body = yield* response.body; - return yield* new FunctionsApiStatusError({ - status: response.status, - message: `unexpected bulk update status ${response.status}: ${body}`, - }); - }); - yield* attempt.pipe( - Effect.retry({ - schedule: transientApiRetrySchedule, - while: (error) => !(error instanceof FunctionsApiStatusError) || error.status >= 500, - }), - ); + if (attempt < 3) { + yield* Effect.sleep(Duration.millis(1_000 * 2 ** attempt)); + } + } + return yield* Effect.fail(lastError ?? new Error("failed to bulk update")); }); const upsertBundledFunction = Effect.fn("functions.deploy.upsertFunction")(function* ( @@ -1951,10 +1816,10 @@ const upsertBundledFunction = Effect.fn("functions.deploy.upsertFunction")(funct bundled: BundledFunction, exists: boolean, ) { - const shouldUpdateRef = yield* Ref.make(exists); + let shouldUpdate = exists; + let lastError: Error | FunctionsApiStatusError | undefined; - const attempt = Effect.gen(function* () { - const shouldUpdate = yield* Ref.get(shouldUpdateRef); + for (let attempt = 0; attempt <= 3; attempt += 1) { const action = shouldUpdate ? "update" : "create"; const updateInput = { ref: projectRef, @@ -1980,50 +1845,52 @@ const upsertBundledFunction = Effect.fn("functions.deploy.upsertFunction")(funct }) : api.executeRaw(operationDefinitions.v1CreateAFunction, createInput); const response = yield* request.pipe( - Effect.mapError((error) => mapTransportError(`failed to ${action} function`, error)), + Effect.map((value) => ({ success: true as const, value })), + Effect.catch((error) => + Effect.succeed({ + success: false as const, + error: mapTransportError(`failed to ${action} function`, error), + }), + ), ); - const expectedStatus = shouldUpdate ? 200 : 201; - if (response.status === expectedStatus) { - // A success status with a malformed / unexpected JSON body is an - // API-response problem, not a transport failure — surface it via - // FunctionsApiStatusError so it classifies as api_status not network. - const body = yield* response.text.pipe(Effect.orElseSucceed(() => "")); - return yield* Effect.try({ - // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- Schema.fromJsonString discards the native SyntaxError on malformed JSON (see decodeJsoncBytes); this raw JSON.parse keeps that message for the decode-error branch below - try: () => decodeDeployFunctionResponse(JSON.parse(body)), - catch: (error) => - new FunctionsApiStatusError({ - status: response.status, - message: `failed to read function response: ${error instanceof Error ? error.message : String(error)}`, - decode: true, - }), + if (response.success) { + const expectedStatus = shouldUpdate ? 200 : 201; + if (response.value.status === expectedStatus) { + // A success status with a malformed / unexpected JSON body is an + // API-response problem, not a transport failure — surface it via + // FunctionsApiStatusError so it classifies as api_status not network. + const body = yield* response.value.text.pipe(Effect.orElseSucceed(() => "")); + return yield* Effect.try({ + try: () => decodeDeployFunctionResponse(JSON.parse(body)), + catch: (error) => + new FunctionsApiStatusError({ + status: response.value.status, + message: `failed to read function response: ${error instanceof Error ? error.message : String(error)}`, + decode: true, + }), + }); + } + + const body = yield* response.value.text.pipe(Effect.orElseSucceed(() => "")); + if (!shouldUpdate && body.includes("Duplicated function slug")) { + shouldUpdate = true; + } + lastError = new FunctionsApiStatusError({ + status: response.value.status, + message: `unexpected ${action} function status ${response.value.status}: ${body}`, + notFoundIsInvalidInput: shouldUpdate, }); + } else { + lastError = response.error; } - const body = yield* response.text.pipe(Effect.orElseSucceed(() => "")); - let nextShouldUpdate = shouldUpdate; - if (!shouldUpdate && body.includes("Duplicated function slug")) { - nextShouldUpdate = true; - yield* Ref.set(shouldUpdateRef, true); + if (attempt < 3) { + yield* Effect.sleep(Duration.millis(500 * 2 ** attempt)); } - return yield* new FunctionsApiStatusError({ - status: response.status, - message: `unexpected ${action} function status ${response.status}: ${body}`, - notFoundIsInvalidInput: nextShouldUpdate, - }); - }); + } - // Every failure here is retried up to the bound, except a decode failure on an - // already-successful status: an unexpected status can still flip `shouldUpdateRef` (a - // `Duplicated function slug` 4xx) and succeed on the retried request, but retrying a malformed - // 200/201 body would only repeat the same write. - return yield* attempt.pipe( - Effect.retry({ - schedule: upsertRetrySchedule, - while: (error) => !(error instanceof FunctionsApiStatusError) || error.decode !== true, - }), - ); + return yield* Effect.fail(lastError ?? new Error("failed to upsert function")); }); const deleteRemoteFunction = Effect.fn("functions.deploy.deleteFunction")(function* ( @@ -2042,38 +1909,44 @@ const deleteRemoteFunction = Effect.fn("functions.deploy.deleteFunction")(functi return; } const body = yield* response.text.pipe(Effect.orElseSucceed(() => "")); - return yield* new FunctionsApiStatusError({ - status: response.status, - message: `unexpected delete function status ${response.status}: ${body}`, - }); + return yield* Effect.fail( + new FunctionsApiStatusError({ + status: response.status, + message: `unexpected delete function status ${response.status}: ${body}`, + }), + ); }); export const discoverFunctionSlugs = Effect.fn("functions.deploy.discoverSlugs")(function* ( projectRoot: string, configDeclaredFunctions: Readonly>, ) { - const fs = yield* FileSystem.FileSystem; const functionsDir = join(projectRoot, SUPABASE_FUNCTIONS_DIR); const slugs: string[] = []; - const entries = yield* fs.readDirectory(functionsDir).pipe( - Effect.map(Option.some), - Effect.catchTag("PlatformError", (error) => - error.reason._tag === "NotFound" - ? Effect.succeed(Option.none>()) - : Effect.fail(nativePlatformFailure(error, functionsDir)), - ), + const entries = yield* Effect.tryPromise({ + try: () => readdir(functionsDir, { withFileTypes: true }), + catch: (cause) => (cause instanceof Error ? cause : new Error(String(cause))), + }).pipe( + Effect.catch((error) => { + return "code" in error && error.code === "ENOENT" + ? Effect.succeed(undefined) + : Effect.fail(error); + }), ); - if (Option.isSome(entries)) { - // The platform `FileSystem` has no Dirent-returning `readdir`, so there's no cheap - // `isDirectory() || isSymbolicLink()` filter to check first; the entrypoint check below - // already requires `/index.ts` to exist, which rules out a non-directory entry the - // same way a separate directory check would. - for (const slug of [...entries.value].sort((left, right) => left.localeCompare(right))) { + if (entries !== undefined) { + for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { + if (!entry.isDirectory() && !entry.isSymbolicLink()) { + continue; + } + const slug = entry.name; if (validateFunctionSlugMessage(slug) !== undefined) { continue; } - if (yield* isFile(fs, defaultFunctionEntrypoint(functionsDir, slug))) { + const hasDefaultEntrypoint = yield* Effect.promise(() => + isFile(defaultFunctionEntrypoint(functionsDir, slug)), + ); + if (hasDefaultEntrypoint) { slugs.push(slug); } } @@ -2106,14 +1979,13 @@ export const resolveFunctionConfigs = Effect.fn("functions.deploy.resolveConfigs readonly noVerifyJwtOverride: Option.Option; }) { const output = yield* Output; - const fs = yield* FileSystem.FileSystem; const functionsDir = join(input.projectRoot, SUPABASE_FUNCTIONS_DIR); const seenDeprecatedImportMap = new Set(); const seenFallbackImportMap = new Set(); const resolved: ResolvedDeployFunctionConfig[] = []; const fallbackImportMapPath = join(functionsDir, "import_map.json"); - const fallbackExists = yield* isFile(fs, fallbackImportMapPath); + const fallbackExists = yield* Effect.promise(() => isFile(fallbackImportMapPath)); const importMapOverride = Option.match(input.importMapOverride, { onNone: () => "", @@ -2168,11 +2040,11 @@ export const resolveFunctionConfigs = Effect.fn("functions.deploy.resolveConfigs const denoJsonc = join(functionDir, "deno.jsonc"); const deprecatedImportMap = join(functionDir, "import_map.json"); - if (yield* isFile(fs, denoJson)) { + if (yield* Effect.promise(() => isFile(denoJson))) { importMap = denoJson; - } else if (yield* isFile(fs, denoJsonc)) { + } else if (yield* Effect.promise(() => isFile(denoJsonc))) { importMap = denoJsonc; - } else if (yield* isFile(fs, deprecatedImportMap)) { + } else if (yield* Effect.promise(() => isFile(deprecatedImportMap))) { importMap = deprecatedImportMap; seenDeprecatedImportMap.add(slug); } else if (fallbackExists) { @@ -2242,7 +2114,9 @@ const deployViaApi = Effect.fn("functions.deploy.viaApi")(function* ( } if (enabled.length === 0) { - return yield* new NoFunctionsToDeployError({ message: "All Functions are up to date." }); + return yield* Effect.fail( + new NoFunctionsToDeployError({ message: "All Functions are up to date." }), + ); } const remoteBySlug = enabled.some((config) => config.verifyJwt === undefined) @@ -2424,7 +2298,9 @@ const pruneFunctions = Effect.fn("functions.deploy.prune")(function* ( ].join("\n")}\n\n`; const confirmed = yield* promptYesNo(output, yes, prompt, false); if (!confirmed) { - return yield* new FunctionDeployCancelledError({ message: CONTEXT_CANCELED_MESSAGE }); + return yield* Effect.fail( + new FunctionDeployCancelledError({ message: CONTEXT_CANCELED_MESSAGE }), + ); } for (const slug of toDelete) { @@ -2433,201 +2309,209 @@ const pruneFunctions = Effect.fn("functions.deploy.prune")(function* ( } }); -export const deployFunctions = Effect.fn("functions.deploy")(function* < - ResolveError, - ResolveRequirements, ->( +export function deployFunctions( flags: FunctionsDeployFlags, dependencies: DeployFunctionsDependencies, ) { - const output = yield* Output; - const styleIdentifier = dependencies.styleIdentifier ?? ((text: string) => text); - const styleEmphasis = dependencies.styleEmphasis ?? ((text: string) => text); - const commandPath = ["functions", "deploy"] as const; - // Presence-based (true for `--use-api=false`, not just bare `--use-api`) — used only for - // the mutual-exclusivity check below. Behavior branches (bundler routing, --jobs guard) key - // off the resolved `flags.useApi` value instead. - const explicitUseApi = hasExplicitLongFlag(dependencies.rawArgs, commandPath, "use-api"); - const explicitUseDocker = hasExplicitLongFlag(dependencies.rawArgs, commandPath, "use-docker"); - const explicitLegacyBundle = hasExplicitLongFlag( - dependencies.rawArgs, - commandPath, - "legacy-bundle", - ); + return Effect.gen(function* () { + const output = yield* Output; + const styleIdentifier = dependencies.styleIdentifier ?? ((text: string) => text); + const styleEmphasis = dependencies.styleEmphasis ?? ((text: string) => text); + const commandPath = ["functions", "deploy"] as const; + // Presence-based (true for `--use-api=false`, not just bare `--use-api`) — used only for + // the mutual-exclusivity check below. Behavior branches (bundler routing, --jobs guard) key + // off the resolved `flags.useApi` value instead. + const explicitUseApi = hasExplicitLongFlag(dependencies.rawArgs, commandPath, "use-api"); + const explicitUseDocker = hasExplicitLongFlag(dependencies.rawArgs, commandPath, "use-docker"); + const explicitLegacyBundle = hasExplicitLongFlag( + dependencies.rawArgs, + commandPath, + "legacy-bundle", + ); - const changedModes = [ - explicitUseApi ? "use-api" : undefined, - explicitUseDocker ? "use-docker" : undefined, - explicitLegacyBundle ? "legacy-bundle" : undefined, - ].filter((flag): flag is string => flag !== undefined); + const changedModes = [ + explicitUseApi ? "use-api" : undefined, + explicitUseDocker ? "use-docker" : undefined, + explicitLegacyBundle ? "legacy-bundle" : undefined, + ].filter((flag): flag is string => flag !== undefined); - if (changedModes.length > 1) { - return yield* new ConflictingFunctionDeployFlagsError({ - message: cobraMutuallyExclusiveErrorMessage(FUNCTIONS_BUNDLER_MUTEX_GROUP, changedModes), - }); - } + if (changedModes.length > 1) { + return yield* Effect.fail( + new ConflictingFunctionDeployFlagsError({ + message: cobraMutuallyExclusiveErrorMessage(FUNCTIONS_BUNDLER_MUTEX_GROUP, changedModes), + }), + ); + } - // `--use-api=false` alone must not force the API path — it should fall through to whatever - // `--use-docker`/`--legacy-bundle` already resolved to. - const useLocalBundler = !flags.useApi && (flags.useDocker || flags.legacyBundle); - const configuredJobs = Option.getOrElse(flags.jobs, () => 1); - const jobs = configuredJobs === 0 ? 1 : configuredJobs; - // Keyed on the resolved `--use-api` value alone, not on whether local bundling - // (Docker/legacy-bundle) is in play. - if (!flags.useApi && jobs > 1) { - return yield* new FunctionDeployJobsRequiresApiError({ - message: "--jobs must be used together with --use-api", - }); - } + // `--use-api=false` alone must not force the API path — it should fall through to whatever + // `--use-docker`/`--legacy-bundle` already resolved to. + const useLocalBundler = !flags.useApi && (flags.useDocker || flags.legacyBundle); + const configuredJobs = Option.getOrElse(flags.jobs, () => 1); + const jobs = configuredJobs === 0 ? 1 : configuredJobs; + // Keyed on the resolved `--use-api` value alone, not on whether local bundling + // (Docker/legacy-bundle) is in play. + if (!flags.useApi && jobs > 1) { + return yield* Effect.fail(new Error("--jobs must be used together with --use-api")); + } - const projectRef = yield* dependencies.resolveProjectRef(flags.projectRef); - // `@supabase/config` merges the matching `[remotes.*]` block over the base config, so this - // already reflects any remote function/edge_runtime overrides, through the same - // `Config.Validate`/dotenv/env-override pipeline `start`/`stop`/`status` use (see - // `functions-config.ts`). Must precede the slug-validation loop below, so an invalid - // `config.toml` is reported ahead of a malformed slug when both are wrong. - const context = yield* loadFunctionsCliConfig({ - projectRoot: dependencies.projectRoot, - projectRef, - goConfigCompat: dependencies.goConfigCompat, - }); + const projectRef = yield* dependencies.resolveProjectRef(flags.projectRef); + // `@supabase/config` merges the matching `[remotes.*]` block over the base config, so this + // already reflects any remote function/edge_runtime overrides, through the same + // `Config.Validate`/dotenv/env-override pipeline `start`/`stop`/`status` use (see + // `functions-config.ts`). Must precede the slug-validation loop below, so an invalid + // `config.toml` is reported ahead of a malformed slug when both are wrong. + const context = yield* loadFunctionsCliConfig({ + projectRoot: dependencies.projectRoot, + projectRef, + goConfigCompat: dependencies.goConfigCompat, + }); - if (flags.functionNames.length > 0) { - for (const slug of flags.functionNames) { - yield* validateDeploySlug(slug); + if (flags.functionNames.length > 0) { + for (const slug of flags.functionNames) { + yield* validateDeploySlug(slug); + } } - } - const noVerifyJwtOverride = explicitBooleanFlag( - dependencies.rawArgs, - ["functions", "deploy"], - "no-verify-jwt", - flags.noVerifyJwt, - ); - // `--debug=false` must resolve to `false` — a plain presence check would get that backwards - // (same rule as `download.ts`'s own `--debug` read). - const debugEnabled = explicitBooleanLongFlag(dependencies.rawArgs, "debug") ?? false; - const deployConfig = context.loaded?.config; - const edgeRuntimeVersion = yield* resolveEdgeRuntimeVersion( - context.denoVersion, - dependencies.edgeRuntimeVersion, - ); - const configFunctions = yield* inferFunctionsManifest({ - cwd: dependencies.projectRoot, - config: deployConfig, - // Matches `loadFunctionsCliConfig`'s own options above: no ancestor directory is searched - // past `dependencies.projectRoot` for either load, so they can never resolve two - // different projects. - search: dependencies.goConfigCompat === undefined, - }); - const configDeclaredFunctions = deployConfig?.functions ?? {}; - const rawConfigFunctions = rawFunctionConfigRecord(context.loaded?.document); - yield* validateConfigFunctionSlugs(configDeclaredFunctions); - const slugs = - flags.functionNames.length > 0 - ? [...flags.functionNames] - : yield* discoverFunctionSlugs(dependencies.projectRoot, configDeclaredFunctions); - - if (slugs.length === 0) { - return yield* new NoFunctionsToDeployError({ - // Styling is text-mode only: in `--output-format json`/`stream-json` this message - // lands in the structured error payload, which must stay free of ANSI escapes. - message: `No Functions specified or found in ${ - output.format === "text" ? styleEmphasis(SUPABASE_FUNCTIONS_DIR) : SUPABASE_FUNCTIONS_DIR - }`, + const noVerifyJwtOverride = explicitBooleanFlag( + dependencies.rawArgs, + ["functions", "deploy"], + "no-verify-jwt", + flags.noVerifyJwt, + ); + // `--debug=false` must resolve to `false` — a plain presence check would get that backwards + // (same rule as `download.ts`'s own `--debug` read). + const debugEnabled = explicitBooleanLongFlag(dependencies.rawArgs, "debug") ?? false; + const deployConfig = context.loaded?.config; + const edgeRuntimeVersion = yield* resolveEdgeRuntimeVersion( + context.denoVersion, + dependencies.edgeRuntimeVersion, + ); + const configFunctions = yield* inferFunctionsManifest({ + cwd: dependencies.projectRoot, + config: deployConfig, + // Matches `loadFunctionsCliConfig`'s own options above: no ancestor directory is searched + // past `dependencies.projectRoot` for either load, so they can never resolve two + // different projects. + search: dependencies.goConfigCompat === undefined, }); - } + const configDeclaredFunctions = deployConfig?.functions ?? {}; + const rawConfigFunctions = rawFunctionConfigRecord(context.loaded?.document); + yield* validateConfigFunctionSlugs(configDeclaredFunctions); + const slugs = + flags.functionNames.length > 0 + ? [...flags.functionNames] + : yield* discoverFunctionSlugs(dependencies.projectRoot, configDeclaredFunctions); + + if (slugs.length === 0) { + return yield* Effect.fail( + new NoFunctionsToDeployError({ + // Styling is text-mode only: in `--output-format json`/`stream-json` this message + // lands in the structured error payload, which must stay free of ANSI escapes. + message: `No Functions specified or found in ${ + output.format === "text" + ? styleEmphasis(SUPABASE_FUNCTIONS_DIR) + : SUPABASE_FUNCTIONS_DIR + }`, + }), + ); + } - const uniqueSlugs = [...new Set(slugs)]; - const configs = yield* resolveFunctionConfigs({ - slugs: uniqueSlugs, - cwd: dependencies.flagCwd, - projectRoot: dependencies.projectRoot, - supabaseDir: dependencies.supabaseDir, - configFunctions, - configDeclaredFunctions, - rawConfigFunctions, - importMapOverride: flags.importMap, - noVerifyJwtOverride, - }); - const dashboardUrl = `${dependencies.dashboardUrl}/project/${projectRef}/functions`; + const uniqueSlugs = [...new Set(slugs)]; + const configs = yield* resolveFunctionConfigs({ + slugs: uniqueSlugs, + cwd: dependencies.flagCwd, + projectRoot: dependencies.projectRoot, + supabaseDir: dependencies.supabaseDir, + configFunctions, + configDeclaredFunctions, + rawConfigFunctions, + importMapOverride: flags.importMap, + noVerifyJwtOverride, + }); + const dashboardUrl = `${dependencies.dashboardUrl}/project/${projectRef}/functions`; - const deployWithApi = deployViaApi( - projectRef, - dependencies.projectRoot, - configs, - dependencies.api, - jobs, - ).pipe( - Effect.as(true), - Effect.catchIf(Schema.is(NoFunctionsToDeployError), (error) => - (output.format === "text" - ? output.raw(`${error.message}\n`, "stderr") - : output.success(error.message, { - project_ref: projectRef, - functions: uniqueSlugs, - dashboard_url: dashboardUrl, - }) - ).pipe(Effect.as(false)), - ), - ); + const deployWithApi = deployViaApi( + projectRef, + dependencies.projectRoot, + configs, + dependencies.api, + jobs, + ).pipe( + Effect.as(true), + Effect.catchIf( + (error): error is NoFunctionsToDeployError => error instanceof NoFunctionsToDeployError, + (error) => + (output.format === "text" + ? output.raw(`${error.message}\n`, "stderr") + : output.success(error.message, { + project_ref: projectRef, + functions: uniqueSlugs, + dashboard_url: dashboardUrl, + }) + ).pipe(Effect.as(false)), + ), + ); - const styleWarning = dependencies.styleWarning ?? ((text: string) => text); - const deployed = useLocalBundler - ? yield* Effect.gen(function* () { - if (!(yield* isDockerRunning())) { - yield* output.raw(`${styleWarning("WARNING:")} Docker is not running\n`, "stderr"); - return yield* deployWithApi; - } + const styleWarning = dependencies.styleWarning ?? ((text: string) => text); + const deployed = useLocalBundler + ? yield* Effect.gen(function* () { + if (!(yield* isDockerRunning())) { + yield* output.raw(`${styleWarning("WARNING:")} Docker is not running\n`, "stderr"); + return yield* deployWithApi; + } - // `lastExplicitLongFlagValue` preserves the "explicitly cleared" vs "never touched" - // distinction `resolveDockerNetworkMode` needs — see that function's own doc comment. - // `SUPABASE_NETWORK_ID` (env or project dotenv) is CLI-only, `undefined` for library - // callers. - const networkMode = resolveDockerNetworkMode({ - explicit: lastExplicitLongFlagValue(dependencies.rawArgs, [], "network-id"), - envOverride: - context.projectEnvValues === undefined - ? undefined - : viperEnvStringWithProjectFallback("SUPABASE_NETWORK_ID", context.projectEnvValues), - projectId: context.projectId, - }); - yield* deployViaDocker({ - projectId: context.projectId, - projectRef, - edgeRuntimeVersion, - functionsDir: join(dependencies.projectRoot, SUPABASE_FUNCTIONS_DIR), - configs, - api: dependencies.api, - networkMode, - verbose: debugEnabled, - styleEmphasis, - projectEnvValues: context.projectEnvValues, - }); - return true; - }) - : yield* deployWithApi; + // `lastExplicitLongFlagValue` preserves the "explicitly cleared" vs "never touched" + // distinction `resolveDockerNetworkMode` needs — see that function's own doc comment. + // `SUPABASE_NETWORK_ID` (env or project dotenv) is CLI-only, `undefined` for library + // callers. + const networkMode = resolveDockerNetworkMode({ + explicit: lastExplicitLongFlagValue(dependencies.rawArgs, [], "network-id"), + envOverride: + context.projectEnvValues === undefined + ? undefined + : viperEnvStringWithProjectFallback( + "SUPABASE_NETWORK_ID", + context.projectEnvValues, + ), + projectId: context.projectId, + }); + yield* deployViaDocker({ + projectId: context.projectId, + projectRef, + edgeRuntimeVersion, + functionsDir: join(dependencies.projectRoot, SUPABASE_FUNCTIONS_DIR), + configs, + api: dependencies.api, + networkMode, + verbose: debugEnabled, + styleEmphasis, + projectEnvValues: context.projectEnvValues, + }); + return true; + }) + : yield* deployWithApi; - if (!deployed) { - return; - } + if (!deployed) { + return; + } - if (output.format === "text") { - // Joins the raw `slugs` list, not the deduped set, so `functions deploy foo foo` prints - // "foo, foo". - yield* output.raw( - `Deployed Functions on project ${styleIdentifier(projectRef)}: ${slugs.join(", ")}\n`, - ); - yield* output.raw(`You can inspect your deployment in the Dashboard: ${dashboardUrl}\n`); - } else { - yield* output.success("Deployed Functions.", { - project_ref: projectRef, - functions: uniqueSlugs, - dashboard_url: dashboardUrl, - }); - } + if (output.format === "text") { + // Joins the raw `slugs` list, not the deduped set, so `functions deploy foo foo` prints + // "foo, foo". + yield* output.raw( + `Deployed Functions on project ${styleIdentifier(projectRef)}: ${slugs.join(", ")}\n`, + ); + yield* output.raw(`You can inspect your deployment in the Dashboard: ${dashboardUrl}\n`); + } else { + yield* output.success("Deployed Functions.", { + project_ref: projectRef, + functions: uniqueSlugs, + dashboard_url: dashboardUrl, + }); + } - if (flags.prune) { - yield* pruneFunctions(projectRef, configs, dependencies.api, dependencies.yes ?? false); - } -}); + if (flags.prune) { + yield* pruneFunctions(projectRef, configs, dependencies.api, dependencies.yes ?? false); + } + }).pipe(Effect.withSpan("functions.deploy")); +} diff --git a/apps/cli/src/shared/functions/deploy.unit.test.ts b/apps/cli/src/shared/functions/deploy.unit.test.ts index b3c903b24c..eff5003774 100644 --- a/apps/cli/src/shared/functions/deploy.unit.test.ts +++ b/apps/cli/src/shared/functions/deploy.unit.test.ts @@ -120,7 +120,9 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: (message) => Effect.sync(() => warnings.push(message)), + onWarning: async (message) => { + warnings.push(message); + }, }); expect(binds.some((bind) => bind.hostPath === vendorIndexPath)).toBe(true); @@ -144,7 +146,7 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil let caught: unknown; try { await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: () => Effect.void, + onWarning: async () => {}, }); } catch (error) { caught = error; @@ -174,7 +176,9 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: (message) => Effect.sync(() => warnings.push(message)), + onWarning: async (message) => { + warnings.push(message); + }, }); expect(binds.some((bind) => formatDockerBind(bind).includes("extra.ts"))).toBe(false); @@ -197,7 +201,9 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: (message) => Effect.sync(() => warnings.push(message)), + onWarning: async (message) => { + warnings.push(message); + }, }); expect(binds.some((bind) => bind.hostPath === vendorIndexPath)).toBe(true); @@ -217,7 +223,9 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: (message) => Effect.sync(() => warnings.push(message)), + onWarning: async (message) => { + warnings.push(message); + }, }); const matches = warnings.filter( @@ -247,7 +255,9 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: (message) => Effect.sync(() => warnings.push(message)), + onWarning: async (message) => { + warnings.push(message); + }, }); expect(binds.some((bind) => formatDockerBind(bind).includes("index.mjs/core"))).toBe(false); @@ -288,7 +298,9 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: (message) => Effect.sync(() => warnings.push(message)), + onWarning: async (message) => { + warnings.push(message); + }, }); expect(binds.some((bind) => formatDockerBind(bind).includes("index.mjs"))).toBe(false); @@ -319,7 +331,9 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil const warnings: Array = []; const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: (message) => Effect.sync(() => warnings.push(message)), + onWarning: async (message) => { + warnings.push(message); + }, skipMissingImportMapTargets: true, }); @@ -345,7 +359,9 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: (message) => Effect.sync(() => warnings.push(message)), + onWarning: async (message) => { + warnings.push(message); + }, skipMissingImportMapTargets: true, }); @@ -404,7 +420,9 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil await writeFile(scopeDependency, 'export const util = "thing";\n'); const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: (message) => Effect.sync(() => warnings.push(message)), + onWarning: async (message) => { + warnings.push(message); + }, }); const hostPaths = binds.map((bind) => bind.hostPath); @@ -443,7 +461,9 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: (message) => Effect.sync(() => warnings.push(message)), + onWarning: async (message) => { + warnings.push(message); + }, }); const functionsBinds = binds.filter((bind) => bind.containerPath === resolve(functionsDir)); @@ -475,7 +495,9 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: (message) => Effect.sync(() => warnings.push(message)), + onWarning: async (message) => { + warnings.push(message); + }, }); expect(warnings).toEqual([]); @@ -499,7 +521,9 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: (message) => Effect.sync(() => warnings.push(message)), + onWarning: async (message) => { + warnings.push(message); + }, }); expect(binds.some((bind) => bind.hostPath === vendorIndexPath)).toBe(true); @@ -527,7 +551,9 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { const binds = await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: (message) => Effect.sync(() => warnings.push(message)), + onWarning: async (message) => { + warnings.push(message); + }, }); expect(binds.some((bind) => bind.hostPath === modPath)).toBe(true); @@ -556,7 +582,9 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: (message) => Effect.sync(() => warnings.push(message)), + onWarning: async (message) => { + warnings.push(message); + }, skipMissingImportMapTargets: true, }); @@ -589,7 +617,9 @@ describe("buildDockerBinds — import-map key matching (spec-strict) and the fil try { await runBuildDockerBinds("test-project", functionsDir, outputDir, config, { - onWarning: (message) => Effect.sync(() => warnings.push(message)), + onWarning: async (message) => { + warnings.push(message); + }, }); expect(warnings.some((warning) => warning.includes("nested"))).toBe(false); diff --git a/apps/cli/src/shared/functions/serve.ts b/apps/cli/src/shared/functions/serve.ts index fd4b3eeb11..579d61fdd7 100644 --- a/apps/cli/src/shared/functions/serve.ts +++ b/apps/cli/src/shared/functions/serve.ts @@ -1694,7 +1694,7 @@ export const resolveFunctionBindMounts = Effect.fn("functions.resolveFunctionBin skipMissingImportMapTargets: true, onWarning: (message) => { bindWarnings.push(message); - return Effect.void; + return Promise.resolve(); }, })) { binds.add(formatDockerBind(bind)); @@ -1787,7 +1787,7 @@ export const startEdgeRuntimeContainer = Effect.fn("functions.startEdgeRuntimeCo skipMissingImportMapTargets: true, onWarning: (message) => { bindWarnings.push(message); - return Effect.void; + return Promise.resolve(); }, })) { const key = formatDockerBind(bind); diff --git a/apps/cli/tests/helpers/child-process-spawner.ts b/apps/cli/tests/helpers/child-process-spawner.ts index 241cfcccd2..ea6c0e1633 100644 --- a/apps/cli/tests/helpers/child-process-spawner.ts +++ b/apps/cli/tests/helpers/child-process-spawner.ts @@ -29,7 +29,7 @@ const isOneShotSupervisor = (args: ReadonlyArray): boolean => { export function mockChildProcessSpawner( opts: { - exitCode?: number | ((record: SpawnRecord) => number); + exitCode?: number; stdout?: string[]; stderr?: string[]; beforeSpawn?: (record: SpawnRecord) => Effect.Effect; @@ -63,9 +63,10 @@ export function mockChildProcessSpawner( cmd === process.execPath && !isOneShotSupervisor(args) ? "30 seconds" : "10 millis", ); running = false; - const resolvedExitCode = - typeof opts.exitCode === "function" ? opts.exitCode(record) : (opts.exitCode ?? 0); - yield* Deferred.succeed(exitDeferred, ChildProcessSpawner.ExitCode(resolvedExitCode)); + yield* Deferred.succeed( + exitDeferred, + ChildProcessSpawner.ExitCode(opts.exitCode ?? 0), + ); }), ); From add89b88bd6856e53e6eb669c2dbd734b9620936 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 00:05:13 +0200 Subject: [PATCH 055/142] fix(stack): drain only in-flight HTTP work and never resume serving after shutdown starts Shutdown drain stops accepting on every listener and waits only for in-flight HTTP requests and upgraded sockets, so idle pooled TCP connections no longer hold stop and destroy open. Draining is one-way: the host owns the single serving fact and injects it into the owner, and the resume and recover paths are removed. A failed stop or destroy reports its error, keeps the stack registered and exits the owner. Stopping a registered stack with no owner reclaims it under the lease or fails. --- packages/stack/ARCHITECTURE.md | 31 ++-- .../stack/src/HttpProxy.integration.test.ts | 29 ---- packages/stack/src/HttpProxy.ts | 12 +- .../stack/src/Network.integration.test.ts | 35 +++++ packages/stack/src/Network.ts | 72 +++------ packages/stack/src/Owner.ts | 18 +-- packages/stack/src/Proxy.integration.test.ts | 145 ++++------------- packages/stack/src/Proxy.ts | 30 +--- ...ost.container-shutdown.integration.test.ts | 79 ++++------ .../stack/src/StackHost.integration.test.ts | 114 +++----------- packages/stack/src/StackHost.ts | 101 ++++++------ packages/stack/src/effect.integration.test.ts | 148 ++++++++++++++---- packages/stack/src/effect.ts | 34 +++- 13 files changed, 373 insertions(+), 475 deletions(-) diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 48a438ee5a..3606aec693 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -476,8 +476,7 @@ stateDiagram-v2 Serving --> Serving: clients connect or disconnect Serving --> Serving: instances start, stop or sleep Serving --> Draining: host.stop or host.destroy - Draining --> Exited: operations settled and cleanup complete - Draining --> Serving: cleanup failed; report error and retain ownership + Draining --> Exited: cleanup complete, or failed and reported Exited --> [*] ``` @@ -506,7 +505,7 @@ nothing left to update, and any resource claim it already recorded stays availab reconcile. Deletion, a move and an unmount are treated identically, because a stack whose root is unreachable cannot operate regardless of which of the three caused it. -**Unreachable engine.** An owner of a container stack fails startup with a `runtime-unavailable` reason when its first container sweep finds the engine CLI missing or its daemon not listening; permission, TLS, authentication and timeout failures are ordinary startup errors. `destroy` then proceeds without an owner: it takes the free lease, publishing a sweeper record like the orphan sweep, refuses when any stack data directory cannot be deleted by the current user, removes the host data and the registration with its port claims, and returns the shell commands that remove the stack's containers and engine-volume data once the engine runs. `stop` without a live owner already succeeds without contacting the engine. +**Unreachable engine.** An owner of a container stack fails startup with a `runtime-unavailable` reason when its first container sweep finds the engine CLI missing or its daemon not listening; permission, TLS, authentication and timeout failures are ordinary startup errors. `destroy` then proceeds without an owner: it takes the free lease, publishing a sweeper record like the orphan sweep, refuses when any stack data directory cannot be deleted by the current user, removes the host data and the registration with its port claims, and returns the shell commands that remove the stack's containers and engine-volume data once the engine runs. `stop` without a live owner takes the same lease and reclaims the registered stack's leftovers (the orphan-sweep path: containers, plus destruction of a session stack) before succeeding; it fails if another process holds the lease or the cleanup fails, and so needs the engine for a container stack. During Serving, keep the owner alive independently of callers. Sleeping instances still need its public listeners. This is process lifetime management, not automatic service restart or continuous reconciliation. @@ -524,21 +523,27 @@ During Draining: **Connection drain.** Stop and destroy share one drain step, run before any service stops: accept closes on every stack listener, public and internal dependency traffic alike (they are the same -listeners), so no new connection is admitted. Connections already established, including HTTP -keep-alive and a workload's own pooled dependency connections (PostgREST's database pool, for -example), keep flowing until each one closes or a 10 second deadline passes, whichever comes -first. At the deadline every remaining connection is cut. Listener scopes close only afterward, -through the ordinary stop/destroy teardown. Accepted, documented tradeoff: during the drain -window, an admitted request that needs a new dependency connection can fail; the deadline bounds -that window. - -**Successful whole-stack shutdown.** `stack.stop()` reports success only after admitted work settles, every owned live service and command workload has stopped (including native processes, descendants and containers labeled with this stack and data root), stack listeners close, the shutdown request is acknowledged, and the detached host's exit is confirmed. If workload cleanup cannot be confirmed, stop fails and the live host retains ownership for inspection and retry; the stack is not reported stopped. If cleanup succeeds but host exit cannot be confirmed, stop also fails, though the host may already have exited. A delivered SIGTERM or SIGINT follows this cleanup path. Stop preserves stack definitions, service data, caches and saved port assignments. `destroy` follows the same live-workload cleanup, then removes only proven-owned persistent data. +listeners), so no new connection is admitted. Drain then waits, for at most 10 seconds, only for +observable work: HTTP requests in flight and upgraded sockets such as WebSockets. TCP connections +never count, so an idle pooled connection (PostgREST's database pool, for example) cannot delay +shutdown. Services then stop, and listener scopes close afterward through the ordinary +stop/destroy teardown, which cuts whatever remains. Accepted, documented tradeoff: during the +drain window, an admitted request that needs a new dependency connection can fail; the deadline +bounds that window. + +**Draining is one-way.** The host holds the single serving/draining fact; the owner reads it for +admission and the network keeps only its listener accept gate. Once a stop or destroy begins, the +owner never serves again. A failed stop or destroy reports its error, the stack stays registered +with uncertain state, and the owner exits. The next stop or destroy finds no owner and reclaims +the stack under its lease. + +**Successful whole-stack shutdown.** `stack.stop()` reports success only after admitted work settles, every owned live service and command workload has stopped (including native processes, descendants and containers labeled with this stack and data root), stack listeners close, the shutdown request is acknowledged, and the detached host's exit is confirmed. If workload cleanup cannot be confirmed, stop fails and the host exits without resuming service; the stack is not reported stopped and remains registered for the next stop or destroy to reclaim. If cleanup succeeds but host exit cannot be confirmed, stop also fails, though the host may already have exited. A delivered SIGTERM or SIGINT follows this cleanup path. Stop preserves stack definitions, service data, caches and saved port assignments. `destroy` follows the same live-workload cleanup, then removes only proven-owned persistent data. The client captures the live owner PID from the validated identity endpoint or readiness handshake, completes the shutdown request, then performs bounded process-existence checks. An absent PID confirms exit; a permission-denied probe remains inconclusive until the deadline. Failure to confirm exit is a `shutdown-exit` error carrying the PID in its message, even when workload cleanup has already succeeded. This does not require persisted PID records or forceful termination. Caller cancellation ends its wait without cancelling admitted owner cleanup. Callers must not start or restart the same stack concurrently with whole-stack shutdown. In particular, replacing an owner between identity lookup and the shutdown request is outside this guarantee. Parallel stacks with separate identities remain independent. Client disposal and Effect scope closure do not implicitly stop a detached stack; disposable fixtures register explicit destruction or use a session stack, which closing its creating handle destroys. -Returning to Serving after cleanup failure does not undo completed cleanup. Unexpected host death does not resume interrupted operations or restore live service state. Native launchers stop their process groups when the dead host's pipe closes. The next host startup for that stack sweeps its containers without removing volumes or saved definitions, and any host start in the same state root sweeps them in the background, also destroying the stack if it is a session stack. A forced host termination does not guarantee immediate container cleanup. A lost control response is reported as uncertain; do not blindly retry a mutation. +A cleanup failure does not undo completed cleanup. Unexpected host death does not resume interrupted operations or restore live service state. Native launchers stop their process groups when the dead host's pipe closes. The next host startup for that stack sweeps its containers without removing volumes or saved definitions, and any host start in the same state root sweeps them in the background, also destroying the stack if it is a session stack. A forced host termination does not guarantee immediate container cleanup. A lost control response is reported as uncertain; do not blindly retry a mutation. ### Request lifetime is separate from execution lifetime diff --git a/packages/stack/src/HttpProxy.integration.test.ts b/packages/stack/src/HttpProxy.integration.test.ts index abca735b7d..458993ef50 100644 --- a/packages/stack/src/HttpProxy.integration.test.ts +++ b/packages/stack/src/HttpProxy.integration.test.ts @@ -1242,32 +1242,3 @@ it.live( }), ).pipe(Effect.provide(NodeServices.layer)), ); - -it.live("destroys every established connection immediately when cutAll runs", () => - Effect.scoped( - Effect.gen(function* () { - const backend = createServer((_request, response) => response.end("ok")); - const backendAddress = yield* listen(backend); - const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); - yield* proxy.setRoutes([{ id: "api", prefix: "/", target: Effect.succeed(backendAddress) }]); - - const client = yield* connectRaw(proxy.port, proxy.host); - client.write("GET / HTTP/1.1\r\nHost: localhost\r\n\r\n"); - const response = yield* readFullResponse(client); - expect(response.status).toBe(200); - expect(response.body).toBe("ok"); - // The request already finished, so the connection sits idle with no active work. - expect(yield* SubscriptionRef.get(proxy.outstandingConnections)).toBe(0); - - const closed = yield* Effect.callback((resume) => { - const onClose = () => resume(Effect.void); - client.once("close", onClose); - if (client.destroyed) onClose(); - return Effect.sync(() => client.off("close", onClose)); - }).pipe(Effect.timeout("5 seconds"), Effect.forkScoped); - yield* proxy.cutAll; - yield* Fiber.join(closed); - expect(yield* SubscriptionRef.get(proxy.outstandingConnections)).toBe(0); - }), - ).pipe(Effect.provide(NodeServices.layer)), -); diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index 872e29e798..ca5bc61d73 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -45,17 +45,13 @@ export interface HttpProxy { readonly host: string; readonly port: number; readonly setRoutes: (routes: ReadonlyArray) => Effect.Effect; - /** Stops accepting new connections on the listener; established connections keep flowing. */ + /** Stops accepting new connections and closes idle ones; in-flight requests keep flowing. */ readonly stopAccepting: Effect.Effect; - /** Reverses `stopAccepting`: recovers a listener left refusing after a failed drain. */ - readonly resumeAccepting: Effect.Effect; /** * The count of connections with active work, observable until it reaches 0: a request in * flight or an upgraded socket, never an idle keep-alive connection. */ readonly outstandingConnections: SubscriptionRef.SubscriptionRef; - /** Destroys every established connection immediately. */ - readonly cutAll: Effect.Effect; } const hopByHop = new Set([ @@ -555,12 +551,6 @@ export const makeHttpProxy = (options: { accepting = false; server.closeIdleConnections(); }), - resumeAccepting: Effect.sync(() => { - accepting = true; - }), outstandingConnections, - cutAll: Effect.sync(() => { - for (const socket of sockets) socket.destroy(); - }), }; }); diff --git a/packages/stack/src/Network.integration.test.ts b/packages/stack/src/Network.integration.test.ts index 3e40e059b2..8e29f0f067 100644 --- a/packages/stack/src/Network.integration.test.ts +++ b/packages/stack/src/Network.integration.test.ts @@ -678,6 +678,41 @@ it.live("drains past an idle HTTP keep-alive connection without ever reaching th ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); +it.live("drains past an idle TCP connection without ever reaching the deadline", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-drain-tcp-" }); + const state = yield* makeTestState(root); + yield* state.save(stack("stack", "auto")); + const target = yield* backend; + const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); + const service = yield* network.register({ + id: "db", + endpoints: { sql: { ...endpoint(target, Effect.succeed(true)), protocol: "tcp" } }, + }); + yield* service.bind; + const address = yield* service.address("sql", "host"); + const socket = yield* Effect.callback((resume) => { + const connection = Net.createConnection({ host: address.host, port: address.port }); + connection.once("connect", () => resume(Effect.succeed(connection))); + connection.on("error", (cause) => + resume(Effect.fail(new FixtureError({ message: cause.message }))), + ); + return Effect.sync(() => connection.destroy()); + }); + + // A deadline that never fires proves TCP never counts toward drain: an idle pooled + // connection stays open and drain still completes. + yield* network.drain.pipe( + Effect.provideService(Network.ShutdownDrainDeadline, Effect.never), + Effect.timeout("5 seconds"), + ); + expect(socket.destroyed).toBe(false); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + it.live( "lets an in-flight HTTP request finish during drain, then closes its now-idle connection", () => diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index b2834fc0de..a3979cda11 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -20,24 +20,18 @@ import { makeHttpProxy, type HttpProxy, type HttpRoute } from "./HttpProxy.ts"; export type NetworkRuntime = "native" | "docker"; /** - * The shutdown-drain deadline: established connections keep flowing until this event - * completes. An event rather than a duration so a test can gate exactly when it fires. + * The shutdown-drain deadline: in-flight HTTP work keeps flowing until this event completes. + * An event rather than a duration so a test can gate exactly when it fires. */ export const ShutdownDrainDeadline = Context.Reference>( "@supabase/stack/ShutdownDrainDeadline", { defaultValue: () => Effect.sleep("10 seconds") }, ); -/** The accept/outstanding/cut-all surface every listener kind (`TcpListener`, `HttpProxy`) shares. */ +/** What shutdown drain needs from a listener: stop accepting, and for HTTP, the in-flight work to wait on. */ interface ListenerHandle { readonly stopAccepting: Effect.Effect; - readonly resumeAccepting: Effect.Effect; - /** - * What counts as active is listener-specific: every TCP connection, or an HTTP connection only - * while it has a request in flight or is upgraded, excluding idle keep-alive sockets. - */ - readonly outstandingConnections: SubscriptionRef.SubscriptionRef; - readonly cutAll: Effect.Effect; + readonly outstandingConnections?: SubscriptionRef.SubscriptionRef; } type RouteContribution = Pick< @@ -100,20 +94,13 @@ export interface Interface { readonly endpoints: Readonly>; }) => Effect.Effect; /** - * Shutdown drain: closes accept on every stack listener, public and dependency alike, then - * lets established connections keep flowing until each reaches 0 or `ShutdownDrainDeadline` - * elapses, whichever comes first. At the deadline every remaining connection is cut. Listener - * scopes are untouched; the caller closes them afterward through the ordinary stop/destroy path. + * Shutdown drain, one-way: closes accept on every stack listener, public and dependency alike, + * then waits until in-flight HTTP requests and upgraded sockets settle or `ShutdownDrainDeadline` + * elapses, whichever comes first. TCP connections are never waited on. Established connections + * stay open and listener scopes are untouched; the caller stops the services and closes the + * scopes afterward, which destroys whatever remains. */ readonly drain: Effect.Effect; - /** - * Recovers from a shutdown that failed after {@link drain} began: resets the drain flag - * `bind` checks and resumes accepting on every currently tracked listener, so a start/restart - * that follows a failed shutdown binds normally instead of refusing as still draining forever. - * Paired with the owner's own admission recovery; never called while a shutdown might still - * confirm success on its own. - */ - readonly recoverDrain: Effect.Effect; } export class Service extends Context.Service()("@supabase/stack/Network") {} @@ -139,10 +126,11 @@ const makeNetwork = (options: { const owner = yield* Scope.Scope; const gate = yield* Semaphore.make(1); const deferReservations = yield* Ref.make(false); - // Set under `gate` by `drain`, before it snapshots `listeners`; checked under the same gate by - // `bind`, so a bind that hasn't yet acquired the gate when drain starts either completes - // before drain's snapshot (and so is captured by it) or observes this and refuses outright - // so neither slips a new, untracked listener past both the snapshot and the accept cut. + // Set once under `gate` by `drain`, before it snapshots `listeners`, and never cleared; checked + // under the same gate by `bind`, so a bind that hasn't yet acquired the gate when drain starts + // either completes before drain's snapshot (and so is captured by it) or observes this and + // refuses outright so neither slips a new, untracked listener past both the snapshot and the + // accept cut. const draining = yield* Ref.make(false); // Every bound listener (the shared API proxy under "api", every dedicated endpoint under // "id:name"), tracked only for shutdown drain; registration and deregistration are tied to the @@ -270,7 +258,7 @@ const makeNetwork = (options: { } yield* probe(key, host, port); const listener = yield* bindTcp(host, port); - yield* trackListener(key, listener); + yield* trackListener(key, { stopAccepting: listener.stopAccepting }); yield* Effect.forkIn( serveTcp(listener, endpoint.backend, `${id}:${name}`).pipe( Effect.provideService(Scope.Scope, endpointScope), @@ -431,11 +419,11 @@ const makeNetwork = (options: { .releaseStack(options.stackId) .pipe(Effect.mapError((cause) => errorFor("release", cause))), ); - // Stopping accepting can only shrink each listener's outstanding count from here on, so waiting - // for every handle to independently reach 0 is equivalent to waiting for the stack-wide total. - const quiesced = (handle: ListenerHandle) => - SubscriptionRef.changes(handle.outstandingConnections).pipe( - Stream.takeUntil((count) => count === 0), + // Stopping accepting can only shrink each HTTP listener's outstanding count from here on, so + // waiting for every handle to independently reach 0 is equivalent to waiting for the total. + const quiesced = (count: SubscriptionRef.SubscriptionRef) => + SubscriptionRef.changes(count).pipe( + Stream.takeUntil((outstanding) => outstanding === 0), Stream.runDrain, ); const drain = Effect.fn("Network.drain")(function* () { @@ -450,29 +438,18 @@ const makeNetwork = (options: { ), ); yield* Effect.forEach(handles, (handle) => handle.stopAccepting, { discard: true }); - const outcome = yield* Effect.forEach(handles, quiesced, { + const inFlight = handles.flatMap((handle) => + handle.outstandingConnections === undefined ? [] : [handle.outstandingConnections], + ); + const outcome = yield* Effect.forEach(inFlight, quiesced, { concurrency: "unbounded", discard: true, }).pipe( Effect.as("quiesced" as const), Effect.race(deadline.pipe(Effect.as("deadline" as const))), ); - if (outcome === "deadline") - yield* Effect.forEach(handles, (handle) => handle.cutAll, { discard: true }); yield* Effect.annotateCurrentSpan({ listeners: handles.length, outcome }); }); - const recoverDrain = Effect.fn("Network.recoverDrain")(() => - gate.withPermits(1)( - Ref.set(draining, false).pipe( - Effect.andThen(Ref.get(listeners)), - Effect.flatMap((current) => - Effect.forEach([...current.values()], (handle) => handle.resumeAccepting, { - discard: true, - }), - ), - ), - ), - ); return { register, release: release(), @@ -480,7 +457,6 @@ const makeNetwork = (options: { cancelDestroy: Ref.set(deferReservations, false), releaseStack: releaseStack(), drain: drain(), - recoverDrain: recoverDrain(), } satisfies Interface; }); diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index 4b39e58e11..fdb059e74d 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -86,6 +86,11 @@ export interface OwnerOptions { readonly hostGateway?: Container.HostGateway; /** The engine endpoint and identity resolved once at startup; absent for a native stack. */ readonly engineTarget?: Container.EngineTarget; + /** + * Reads whether shutdown has begun. The host owns that one-way fact; an owner built without a + * host (for example a sweep) never drains and so admits all work. + */ + readonly draining?: Effect.Effect; } type OwnerRpcs = RpcGroup.Rpcs; @@ -188,16 +193,9 @@ export interface Interface { * the engine-unreachable backstop is logged, not thrown. */ readonly abandon: Effect.Effect; - /** - * Recovers network admission after a shutdown that failed once drain had already begun; see - * {@link Network.Interface.recoverDrain}. - */ - readonly recoverDraining: Effect.Effect; }; /** Direct probe of the pinned engine target; always `true` for a native stack. */ readonly engineAvailable: Effect.Effect; - readonly setDraining: (draining: boolean) => Effect.Effect; - readonly getServing: Effect.Effect; } export class Service extends Context.Service()("@supabase/stack/Owner") {} @@ -339,8 +337,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { }), ), ); - const draining = yield* Ref.make(false); - const rejectWhileDraining = Ref.get(draining).pipe( + const rejectWhileDraining = (options.draining ?? Effect.succeed(false)).pipe( Effect.flatMap((isDraining) => isDraining ? Effect.fail(new ServiceError({ operation: "draining", message: "Owner is draining" })) @@ -970,11 +967,8 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { definitionGate.withPermits(1), Effect.withSpan("Owner.abandonNamespace", { attributes: { stack_id: stackId } }), ), - recoverDraining: network.recoverDrain, }, engineAvailable, - setDraining: (value) => Ref.set(draining, value), - getServing: Ref.get(draining).pipe(Effect.map((isDraining) => !isDraining)), } satisfies Interface; }); diff --git a/packages/stack/src/Proxy.integration.test.ts b/packages/stack/src/Proxy.integration.test.ts index 56f1248a31..db13c06afa 100644 --- a/packages/stack/src/Proxy.integration.test.ts +++ b/packages/stack/src/Proxy.integration.test.ts @@ -1,16 +1,6 @@ import { NodeHttpClient, NodeHttpServer } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { - Deferred, - Effect, - Exit, - Fiber, - Predicate, - Ref, - Scope, - Stream, - SubscriptionRef, -} from "effect"; +import { Deferred, Effect, Exit, Fiber, Predicate, Ref, Scope } from "effect"; import { HttpClient, HttpClientRequest, @@ -259,7 +249,7 @@ it.live( ); it.live( - "refuses a new connection after stopAccepting while an established connection keeps relaying data through the scoped target until it closes", + "refuses a new connection after stopAccepting while an established connection keeps relaying data through the scoped target", () => Effect.scoped( Effect.gen(function* () { @@ -273,7 +263,6 @@ it.live( const client = yield* connectAndWrite(port, "ping"); expect(yield* readOnceTcp(client)).toBe("ping"); - expect(yield* SubscriptionRef.get(listener.outstandingConnections)).toBe(1); yield* listener.stopAccepting; @@ -300,112 +289,38 @@ it.live( client.write("pong"); expect(yield* readOnceTcp(client)).toBe("pong"); - - const reachedZero = yield* Stream.runHead( - Stream.filter( - SubscriptionRef.changes(listener.outstandingConnections), - (count) => count === 0, - ), - ).pipe(Effect.forkScoped); - yield* Effect.sync(() => client.end()); - yield* Fiber.join(reachedZero); - }), - ), -); - -it.live( - "destroys every established connection immediately when cutAll runs on the production listener", - () => - Effect.scoped( - Effect.gen(function* () { - const backend = yield* echoBackend(); - const listener = yield* bindTcp("127.0.0.1", 0); - if (!Predicate.isTagged(listener.address, "TcpAddress")) - return yield* Effect.die("Expected TCP listener"); - const target = Effect.succeed({ host: backend.host, port: backend.port }); - yield* serveTcp(listener, target, "echo").pipe(Effect.forkScoped); - const port = listener.address.port; - - const client = yield* connectAndWrite(port, "ping"); - expect(yield* readOnceTcp(client)).toBe("ping"); - expect(yield* SubscriptionRef.get(listener.outstandingConnections)).toBe(1); - - const closed = yield* Effect.callback((resume) => { - const onClose = () => resume(Effect.void); - client.once("close", onClose); - if (client.destroyed) onClose(); - return Effect.sync(() => client.off("close", onClose)); - }).pipe(Effect.timeout("5 seconds"), Effect.forkScoped); - // Subscribed before `cutAll` runs: the client's own "close" event is a separate, cross- - // socket signal that reaches this process over the loopback connection, with no ordering - // guarantee relative to the listener's own bookkeeping (observed to lag it on Windows), so - // the listener's count must be awaited on its own observable, never inferred from the - // client's. - const reachedZero = yield* Stream.runHead( - Stream.filter( - SubscriptionRef.changes(listener.outstandingConnections), - (count) => count === 0, - ), - ).pipe(Effect.forkScoped); - yield* listener.cutAll; - yield* Fiber.join(closed); - yield* Fiber.join(reachedZero); - expect(yield* SubscriptionRef.get(listener.outstandingConnections)).toBe(0); }), ), ); -it.live( - "counts, cuts, and tears down a connection that arrives before run installs its handler", - () => - Effect.gen(function* () { - const listenerScope = yield* Scope.make(); - const listener = yield* bindTcp("127.0.0.1", 0).pipe(Scope.provide(listenerScope)); - if (!Predicate.isTagged(listener.address, "TcpAddress")) - return yield* Effect.die("Expected TCP listener"); - const port = listener.address.port; - - const connect = () => - Effect.callback((resume) => { - const socket = new Socket(); - socket.once("connect", () => resume(Effect.succeed(socket))); - socket.connect(port, "127.0.0.1"); - return Effect.void; - }).pipe(Effect.timeout("5 seconds")); - const awaitClose = (socket: Socket) => - Effect.callback((resume) => { - const onClose = () => resume(Effect.void); - socket.once("close", onClose); - if (socket.destroyed) onClose(); - return Effect.sync(() => socket.off("close", onClose)); - }).pipe(Effect.timeout("5 seconds")); - // The server's own acceptance (tracked by `outstandingConnections`) is a separate event from - // the client's own "connect"; subscribed before connecting so neither ordering is assumed. - const awaitOutstanding = (expected: number) => - Stream.runHead( - Stream.filter( - SubscriptionRef.changes(listener.outstandingConnections), - (count) => count === expected, - ), - ).pipe(Effect.timeout("5 seconds")); +it.live("tears down a connection that arrives before run installs its handler", () => + Effect.gen(function* () { + const listenerScope = yield* Scope.make(); + const listener = yield* bindTcp("127.0.0.1", 0).pipe(Scope.provide(listenerScope)); + if (!Predicate.isTagged(listener.address, "TcpAddress")) + return yield* Effect.die("Expected TCP listener"); + const port = listener.address.port; - // `run` is never called on this listener: every connection below is, and stays, queued. - const firstAccepted = yield* awaitOutstanding(1).pipe(Effect.forkScoped); - const first = yield* connect(); - yield* Fiber.join(firstAccepted); - - const firstCut = yield* awaitOutstanding(0).pipe(Effect.forkScoped); - yield* listener.cutAll; - yield* awaitClose(first); - yield* Fiber.join(firstCut); + const connect = Effect.callback((resume) => { + const socket = new Socket(); + socket.once("connect", () => resume(Effect.succeed(socket))); + socket.connect(port, "127.0.0.1"); + return Effect.void; + }).pipe(Effect.timeout("5 seconds")); + const awaitClose = (socket: Socket) => + Effect.callback((resume) => { + const onClose = () => resume(Effect.void); + socket.once("close", onClose); + if (socket.destroyed) onClose(); + return Effect.sync(() => socket.off("close", onClose)); + }).pipe(Effect.timeout("5 seconds")); - // Closing the listener's own scope destroys a still-queued connection and completes rather - // than hanging on `server.close()` waiting for a socket nothing ever destroyed. - const secondAccepted = yield* awaitOutstanding(1).pipe(Effect.forkScoped); - const second = yield* connect(); - yield* Fiber.join(secondAccepted); - const secondClosed = yield* awaitClose(second).pipe(Effect.forkScoped); - yield* Scope.close(listenerScope, Exit.void).pipe(Effect.timeout("5 seconds")); - yield* Fiber.join(secondClosed); - }).pipe(Effect.scoped), + // `run` is never called on this listener: the connection below stays queued. Closing the + // listener's own scope destroys it and completes rather than hanging on `server.close()` + // waiting for a socket nothing ever destroyed. + const queued = yield* connect; + const closed = yield* awaitClose(queued).pipe(Effect.forkScoped); + yield* Scope.close(listenerScope, Exit.void).pipe(Effect.timeout("5 seconds")); + yield* Fiber.join(closed); + }).pipe(Effect.scoped), ); diff --git a/packages/stack/src/Proxy.ts b/packages/stack/src/Proxy.ts index 15c914e78d..9bd17a95a6 100644 --- a/packages/stack/src/Proxy.ts +++ b/packages/stack/src/Proxy.ts @@ -1,5 +1,5 @@ import { NodeSink, NodeStream } from "@effect/platform-node"; -import { Data, Effect, Exit, FiberSet, Scope, Stream, SubscriptionRef } from "effect"; +import { Data, Effect, Exit, FiberSet, Scope, Stream } from "effect"; import type { SocketServer } from "effect/unstable/socket"; import * as Net from "node:net"; import { PortError } from "./Ports.ts"; @@ -24,19 +24,13 @@ export interface TcpListener { ) => Effect.Effect; /** Stops accepting new connections on the listener; established connections keep flowing. */ readonly stopAccepting: Effect.Effect; - /** Reverses `stopAccepting`: recovers a listener left refusing after a failed drain. */ - readonly resumeAccepting: Effect.Effect; - /** The count of established client connections, observable until it reaches 0. */ - readonly outstandingConnections: SubscriptionRef.SubscriptionRef; - /** Destroys every established connection immediately. */ - readonly cutAll: Effect.Effect; } /** * Binds a dedicated public listener and retains the socket until its scope closes. Built directly * on `node:net`, not `NodeSocketServer`, because `NodeSocketServer`'s `run` only exposes a combined - * accept-and-destroy teardown: this separates closing accept (`stopAccepting`) from cutting - * established connections (`cutAll`), with the count of outstanding ones observable in between. + * accept-and-destroy teardown: this separates closing accept (`stopAccepting`) from destroying + * established connections, which happens when the scope closes. */ export const bindTcp = ( host: string, @@ -44,10 +38,9 @@ export const bindTcp = ( ): Effect.Effect => Effect.gen(function* () { const services = yield* Effect.context(); - // Every accepted socket, queued or dispatched, from acceptance until close: the single source - // for the outstanding count, `cutAll`, and teardown, so none of them is ever only half-tracked. + // Every accepted socket, queued or dispatched, from acceptance until close, so teardown + // destroys all of them. const sockets = new Set(); - const outstandingConnections = yield* SubscriptionRef.make(0); let accepting = true; // A connection that arrives before `run` installs its handler is queued, the same as // `NodeSocketServer`'s own listener, so none are dropped in that gap. @@ -64,15 +57,9 @@ export const bindTcp = ( return; } sockets.add(conn); - Effect.runSyncWith(services)( - SubscriptionRef.update(outstandingConnections, (count) => count + 1), - ); conn.once("close", () => { sockets.delete(conn); pending.delete(conn); - Effect.runSyncWith(services)( - SubscriptionRef.update(outstandingConnections, (count) => count - 1), - ); }); onConnection(conn); }); @@ -149,13 +136,6 @@ export const bindTcp = ( stopAccepting: Effect.sync(() => { accepting = false; }), - resumeAccepting: Effect.sync(() => { - accepting = true; - }), - outstandingConnections, - cutAll: Effect.sync(() => { - for (const socket of sockets) socket.destroy(); - }), }; }); diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index 82418da3b7..6cd18960a7 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -583,26 +583,28 @@ const abandonsWhileStopDrains = (trigger: StopTrigger) => yield* client.startService({ id: database.id }); yield* client.readyService({ id: database.id }); const status = yield* client.status({ id: database.id }); - const port = status.endpoints.find((endpoint) => endpoint.name === "sql")?.port; - if (port === undefined) return yield* Effect.die("Missing database sql endpoint"); + if (!status.endpoints.some((endpoint) => endpoint.name === "sql")) + return yield* Effect.die("Missing database sql endpoint"); + const mail = yield* client.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* client.startService({ id: mail.id }); + yield* client.readyService({ id: mail.id }); + const mailPort = (yield* client.status({ id: mail.id })).endpoints.find( + (endpoint) => endpoint.name === "http", + )?.port; + if (mailPort === undefined) return yield* Effect.die("Missing mail http endpoint"); const containerEnvRoot = `${dataRoot}/.container-env`; expect( yield* fs.exists(containerEnvRoot), "the stack's shared container-env scratch directory exists", ).toBe(true); - // Keeps the sql listener's outstanding-connection count above zero for the whole gated - // window, so drain can only resolve through the deadline this test controls, never - // through every connection reaching zero on its own. - const held = yield* Effect.acquireRelease( - Effect.callback((resume) => { - const connection = Net.createConnection({ host: "127.0.0.1", port }); - connection.once("connect", () => resume(Effect.succeed(connection))); - connection.once("error", (cause) => resume(Effect.die(cause))); - return Effect.sync(() => connection.destroy()); - }), - (connection) => Effect.sync(() => connection.destroy()), - ); + // Keeps an HTTP request in flight for the whole gated window, so drain can only resolve + // through the deadline this test controls, never through the work settling on its own. + const held = yield* openPartialMailSend(mailPort, "abandon-gated-inflight"); // Subscribes to the drain-deadline wait's own entry marker before triggering the stop, so // the registration deletion below never races the gate itself. @@ -612,7 +614,7 @@ const abandonsWhileStopDrains = (trigger: StopTrigger) => yield* fs.remove(`${stateRoot}/${stackId}/state.json`); yield* fs.writeFileString(`${gateDir}/release`, ""); - yield* Effect.sync(() => held.destroy()); + held.complete(); yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( Effect.retry({ while: hasReason("owner-exit-pending") }), @@ -919,7 +921,7 @@ it.live.skipIf(process.platform === "win32")( ); it.live.skipIf(process.platform === "win32")( - "keeps a pinned postgres connection serving through a real owner shutdown, and cuts it once the deadline fires", + "closes a pinned postgres connection and exits a real owner without waiting for the drain deadline", () => Effect.scoped( Effect.gen(function* () { @@ -990,43 +992,24 @@ it.live.skipIf(process.platform === "win32")( const sql = Context.get(services, PgClient.PgClient); yield* sql.unsafe("SELECT 1"); - // A second, query-free connection to the same listener: its closure is a push-based - // signal that the deadline's cut already reached the gateway, with no query to race. - const probe = yield* Effect.acquireRelease( - Effect.callback((resume) => { - const connection = Net.createConnection({ host: "127.0.0.1", port }); - connection.once("connect", () => resume(Effect.succeed(connection))); - connection.once("error", (cause) => resume(Effect.die(cause))); - return Effect.sync(() => connection.destroy()); - }), - (connection) => Effect.sync(() => connection.destroy()), - ); - - const waiting = yield* watchEntry(gateDir, "waiting", true); - process.kill(access.endpoint.pid, "SIGTERM"); - yield* waiting.pipe(Effect.timeout("30 seconds")); - - yield* sql.unsafe("SELECT 1"); - - const closed = yield* Effect.callback((resume) => { - if (probe.destroyed) { - resume(Effect.void); - return Effect.void; - } - const onClose = () => resume(Effect.void); - probe.once("close", onClose); - return Effect.sync(() => probe.off("close", onClose)); - }).pipe(Effect.forkChild({ startImmediately: true })); - yield* fs.writeFileString(`${gateDir}/release`, ""); - yield* Fiber.join(closed).pipe(Effect.timeout("30 seconds")); - - const cut = yield* sql.unsafe("SELECT 1").pipe(Effect.timeout("5 seconds"), Effect.exit); - expect(Exit.isFailure(cut), "the connection is cut once the deadline fires").toBe(true); + // The gate is never released: only a drain that ignores the pinned TCP connection lets + // the owner finish, and the connection closes once the services stop. + yield* Effect.callback((resume) => { + const probe = Net.createConnection({ host: "127.0.0.1", port }); + probe.once("connect", () => { + probe.once("close", () => resume(Effect.void)); + process.kill(access.endpoint.pid, "SIGTERM"); + }); + probe.once("error", (cause) => resume(Effect.die(cause))); + return Effect.sync(() => probe.destroy()); + }); yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( Effect.retry({ while: hasReason("owner-exit-pending") }), Effect.timeout("30 seconds"), ); + const cut = yield* sql.unsafe("SELECT 1").pipe(Effect.timeout("5 seconds"), Effect.exit); + expect(Exit.isFailure(cut), "the pinned connection is gone with the owner").toBe(true); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 180_000 }, diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts index ee5b6b3a05..0c4df16eb3 100644 --- a/packages/stack/src/StackHost.integration.test.ts +++ b/packages/stack/src/StackHost.integration.test.ts @@ -64,10 +64,13 @@ const ownerFor = (options: { }) => { const { state, ...layerOptions } = options; return Effect.gen(function* () { + const draining = yield* Deferred.make(); const context = yield* Layer.build( - Owner.layer(layerOptions).pipe(Layer.provide(Layer.succeed(StackNamespace.Service, state))), + Owner.layer({ ...layerOptions, draining: Deferred.isDone(draining) }).pipe( + Layer.provide(Layer.succeed(StackNamespace.Service, state)), + ), ); - return Context.get(context, Owner.Service); + return { ...Context.get(context, Owner.Service), draining }; }); }; @@ -227,7 +230,7 @@ const openPartialMailSend = (port: number, subject: string) => }); const inProcessRuntime = ( - owner: Parameters[0], + owner: Parameters[0] & { readonly draining: Deferred.Deferred }, state: StackNamespace.Interface, root: string, ) => @@ -255,6 +258,7 @@ const inProcessRuntime = ( }, acquired.server, acquired.closeConnections, + owner.draining, ).pipe( Effect.provideService(CommandRunner.Service, Context.get(toolContext, CommandRunner.Service)), ); @@ -361,7 +365,7 @@ it.live("preserves composition outcomes over RPC", () => ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); -it.live("keeps serving when namespace shutdown fails", () => +it.live("stops serving after a failed shutdown and keeps reporting that failure", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -393,72 +397,18 @@ it.live("keeps serving when namespace shutdown fails", () => }, }; const { runtime } = yield* inProcessRuntime(failedOwner, state, root); - const client = yield* ownerClient(runtime.access); - const failure = yield* shutdownOwner(runtime.access, false).pipe(Effect.flip); - expect(failure.message).toContain("cleanup failed"); - yield* client.configureComposition({ members: [], dependencies: [] }); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), -); - -it.live("recovers network admission after a shutdown that fails once drain has begun (F6)", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-host-drain-recovery-" }); - const state = yield* stateFor(`${root}/state`); - const saved = { - id: "stack", - runtime: "native" as const, - identity: { projectRoot: root, branchContext: "main", stackName: "host-drain-recovery" }, - instances: [], - lifetime: "detached" as const, - composition: { members: [], dependencies: [] }, - ports: [], - }; - yield* state.save(saved); - const owner = yield* ownerFor({ - saved, - state, - root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", - }); - const failedOwner = { - ...owner, - namespace: { - ...owner.namespace, - // Lets the real drain run (so `Network`'s own internal draining flag is genuinely set), - // then fails afterward: reproduces "shutdown fails after drain begins" without reaching - // into `Network`'s internals directly. - stop: owner.namespace.stop.pipe( - Effect.andThen( - Effect.fail(new OrchestratorError({ operation: "stop", message: "cleanup failed" })), - ), - ), - }, - }; - const { runtime } = yield* inProcessRuntime(failedOwner, state, root); - const client = yield* ownerClient(runtime.access); const failure = yield* shutdownOwner(runtime.access, false).pipe(Effect.flip); expect(failure.message).toContain("cleanup failed"); - - // F6: a drain that already began during the failed shutdown must not block ordinary - // service recovery forever; starting a service again must bind its listener, not refuse it - // as still draining. - const mail = yield* client.createService({ - service: "mail", - config: {}, - endpoints: { http: { port: "auto" } }, - }); - yield* client.startService({ id: mail.id }); - yield* client.readyService({ id: mail.id }); - yield* shutdownOwner(runtime.access, true); + yield* Deferred.await(runtime.exit).pipe(Effect.timeout("5 seconds")); + expect(yield* Deferred.isDone(owner.draining)).toBe(true); + const again = yield* runtime.shutdown(true).pipe(Effect.flip); + expect(again.message).toContain("Shutdown mode is already selected"); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); it.live( - "freezes admission and confirms cleanup when abandonment claims ownership after a failing stop settles (pass 3, A)", + "closes admission and confirms cleanup when abandonment claims ownership after a failing stop settles", () => Effect.scoped( Effect.gen(function* () { @@ -513,11 +463,10 @@ it.live( yield* Deferred.succeed(stopGate, undefined); expect(Exit.isFailure(yield* Fiber.join(stopFiber).pipe(Effect.exit))).toBe(true); yield* Deferred.await(abandonEntered).pipe(Effect.timeout("2 seconds")); - // Admission stays frozen through abandonment's own cleanup, never left reopened by the - // failing stop's own recovery that preceded it (F2). - expect(yield* owner.getServing, "admission stays frozen while abandonment cleans up").toBe( - false, - ); + expect( + yield* Deferred.isDone(owner.draining), + "admission stays closed while abandonment cleans up", + ).toBe(true); yield* Deferred.succeed(abandonGate, undefined); yield* Fiber.join(abandonFiber).pipe(Effect.timeout("2 seconds")); yield* Deferred.await(runtime.exit).pipe(Effect.timeout("2 seconds")); @@ -525,7 +474,7 @@ it.live( ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); -it.live("reports a destroy failure and leaves the owner running", () => +it.live("reports a destroy failure and the owner exits", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -582,9 +531,9 @@ it.live("reports a destroy failure and leaves the owner running", () => error: expect.stringContaining("data removal refused"), }, ]); - // A failed destroy does not fall back to stop: retry destroy or call `stack stop` instead. - expect(yield* Deferred.isDone(runtime.exit)).toBe(false); - expect(yield* owner.getServing).toBe(true); + // A failed destroy does not fall back to stop: the owner exits and the registered stack + // is reclaimed by the next stop or destroy. + yield* Deferred.await(runtime.exit).pipe(Effect.timeout("5 seconds")); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); @@ -635,7 +584,7 @@ it.live("rejects destroy while stop is in flight", () => ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); -it.live("retains ownership when namespace shutdown defects and retries cleanup", () => +it.live("exits after a namespace shutdown defect without rerunning cleanup", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -669,18 +618,12 @@ it.live("retains ownership when namespace shutdown defects and retries cleanup", }, }; const { runtime } = yield* inProcessRuntime(failedOwner, state, root); - const client = yield* ownerClient(runtime.access); const failure = yield* runtime.shutdown(false).pipe(Effect.exit); expect(Exit.isFailure(failure)).toBe(true); if (Exit.isFailure(failure)) expect(Cause.pretty(failure.cause)).toContain("cleanup failed"); - const http = yield* HttpClient.HttpClient; - const identity = yield* http.get(`http://127.0.0.1:${runtime.endpoint.port}/identity`, { - headers: { authorization: ownerAuthorization(runtime.access.secret) }, - }); - expect(identity.status).toBe(200); - expect(yield* owner.getServing).toBe(true); - yield* client.configureComposition({ members: [], dependencies: [] }); - yield* runtime.shutdown(false); + yield* Deferred.await(runtime.exit).pipe(Effect.timeout("5 seconds")); + // The stop would succeed if it ran again; the failure is final for this owner. + expect(Exit.isFailure(yield* runtime.shutdown(false).pipe(Effect.exit))).toBe(true); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); @@ -1172,14 +1115,9 @@ const abandonedComposition = (prefix: string, destroy: boolean) => cacheRoot: "/tmp/supabase-stack-artifacts", }); const interrupted = yield* Deferred.make(); - const draining = yield* Deferred.make(); const { runtime } = yield* inProcessRuntime( { ...owner, - setDraining: (value: boolean) => - owner - .setDraining(value) - .pipe(Effect.andThen(value ? Deferred.succeed(draining, undefined) : Effect.void)), handlers: { ...owner.handlers, supabaseComposition: (input: Parameters[0]) => @@ -1212,7 +1150,7 @@ const abandonedComposition = (prefix: string, destroy: boolean) => yield* Fiber.interrupt(composition); yield* Deferred.await(interrupted); const shutdown = yield* Effect.forkScoped(shutdownOwner(runtime.access, destroy)); - yield* Deferred.await(draining); + yield* Deferred.await(owner.draining); yield* Deferred.succeed(allow, undefined); yield* Fiber.join(shutdown); yield* Deferred.await(runtime.exit).pipe(Effect.timeout("5 seconds")); diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index fdcbe2886b..7e9ee1bcb6 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -149,10 +149,14 @@ const creatorGone = Effect.callback((resume) => { }); }); -const isOpen = (value: boolean): Effect.Effect => - value - ? Effect.void - : Effect.fail(new StackError({ operation: "host", message: "Stack host is draining" })); +const rejectWhileDraining = (draining: Deferred.Deferred): Effect.Effect => + Deferred.isDone(draining).pipe( + Effect.flatMap((isDraining) => + isDraining + ? Effect.fail(new StackError({ operation: "host", message: "Stack host is draining" })) + : Effect.void, + ), + ); const requestSignal = () => Effect.callback<"SIGTERM" | "SIGINT", never>((resume) => { @@ -195,18 +199,23 @@ export interface StackHostRuntime { readonly exit: Deferred.Deferred; } +/** + * Serves the host. `draining` is the one source of truth for whether shutdown has begun: the host + * completes it when a shutdown begins and never resets it, and the owner reads it for admission. + */ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( ( owner: Owner.Interface, access: HostAccess, server: HttpServer.HttpServer["Service"], closeConnections: Effect.Effect, + draining: Deferred.Deferred, ): Effect.Effect => Effect.gen(function* () { const scope = yield* Scope.Scope; const runner = yield* CommandRunner.Service; const attachments = yield* makeCommandAttachments({ - admit: owner.getServing.pipe(Effect.flatMap(isOpen)), + admit: rejectWhileDraining(draining), run: (input) => "stdin" in input ? runner.run({ @@ -251,17 +260,16 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( Effect.catchCause((cause) => Effect.logError(`Abandoned stack: ${label} failed`, cause)), ); - // Admits `mode`'s cleanup when nothing else is in flight: forks it, tracks it as `current` - // until it settles (success or failure alike), then clears `current` so a later claim — - // for example abandon after an in-flight stop or destroy — can be admitted again. Only - // `stop` and `destroy` can actually fail; their own recovery (reopening admission) runs, - // and `current` is cleared, before `claim` below can ever admit anything else. + // Admits `mode`'s cleanup when nothing else is in flight: begins draining, forks the cleanup + // and tracks it as `current`. Draining never reverses: a failed stop or destroy leaves its + // failed fiber in `current` (repeat calls rejoin the same failure), and the owner exits + // once the response has been delivered; only abandon may replace it. const begin = ( mode: ShutdownMode, response?: ReturnType, ): Effect.Effect> => Effect.gen(function* () { - yield* owner.setDraining(true); + yield* Deferred.succeed(draining, undefined); const responseClosedSignal = response === undefined ? undefined : yield* Deferred.make(); if (response !== undefined && responseClosedSignal !== undefined) @@ -309,26 +317,11 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( yield* runner.cleanup; yield* mode === "stop" ? owner.namespace.stop : owner.namespace.destroy; }).pipe( + Effect.onError(() => finish), Effect.andThen(finish), Effect.mapError((cause) => stackError("shutdown", cause)), ); - const cleanup = body.pipe( - Effect.catchCause((cause) => - mode !== "abandon" - ? owner - .setDraining(false) - // Drain may already have run (and so left the network's own admission - // closed) before this later stage failed; recovering it here keeps a - // start/restart that follows from refusing as still draining. - .pipe( - Effect.andThen(owner.namespace.recoverDraining), - Effect.andThen(gate.withPermits(1)(Ref.set(current, undefined))), - Effect.andThen(Effect.failCause(cause)), - ) - : Effect.failCause(cause), - ), - ); - const fiber = yield* Effect.forkIn(cleanup, scope); + const fiber = yield* Effect.forkIn(body, scope); yield* Ref.set(current, { mode, fiber }); return fiber; }); @@ -545,6 +538,7 @@ export const runStackHost = Effect.fn("StackHost.run")( yield* state.save(register); }), ); + const draining = yield* Deferred.make(); const started = yield* Effect.gen(function* () { const saved = yield* state.read(id); if (saved === undefined) return yield* hostError("startup", "Stack is not registered"); @@ -582,6 +576,7 @@ export const runStackHost = Effect.fn("StackHost.run")( const services = yield* Layer.build( Layer.merge( Owner.layer({ + draining: Deferred.isDone(draining), saved, root: dataRoot, cacheRoot: options.cacheRoot, @@ -617,6 +612,7 @@ export const runStackHost = Effect.fn("StackHost.run")( access, control.server, control.closeConnections, + draining, ).pipe( Effect.provideService( CommandRunner.Service, @@ -667,35 +663,30 @@ export const runStackHost = Effect.fn("StackHost.run")( register === undefined ? Effect.void : state.remove(id).pipe(Effect.ignore), ), ); - while (true) { - const event = yield* Deferred.await(started.exit).pipe( - Effect.map(() => "done" as const), - Effect.raceFirst(Queue.take(events)), + const event = yield* Deferred.await(started.exit).pipe( + Effect.map(() => "done" as const), + Effect.raceFirst(Queue.take(events)), + ); + if (event === "creator-gone") { + yield* started.shutdown(true).pipe( + Effect.tapCause((cause) => Effect.logError("Session stack destroy failed", cause)), + Effect.ignore, ); - if (event === "creator-gone") { - yield* started.shutdown(true).pipe( - Effect.tapCause((cause) => Effect.logError("Session stack destroy failed", cause)), - Effect.ignore, - ); - break; - } - if (event === "registration-gone") { - yield* started.abandon; - break; - } - if (event !== "done") { - const shutdownSucceeded = yield* started.shutdown(false).pipe( - Effect.tapCause((cause) => Effect.logError("Stack shutdown failed", cause)), - Effect.matchCause({ onSuccess: () => true, onFailure: () => false }), - ); - if (!shutdownSucceeded) continue; - } - // A registration deletion that arrives while a data-preserving stop is already running - // only reaches `events` as a queued, now-unread `registration-gone`: a direct re-read - // through the same state API the poll uses catches it before the owner exits. - if (yield* registrationConfirmedGone(state, id)) yield* started.abandon; - break; + return; + } + if (event === "registration-gone") { + yield* started.abandon; + return; } + if (event !== "done") + yield* started.shutdown(false).pipe( + Effect.tapCause((cause) => Effect.logError("Stack shutdown failed", cause)), + Effect.ignore, + ); + // A registration deletion that arrives while a data-preserving stop is already running + // only reaches `events` as a queued, now-unread `registration-gone`: a direct re-read + // through the same state API the poll uses catches it before the owner exits. + if (yield* registrationConfirmedGone(state, id)) yield* started.abandon; }), ).pipe( Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp)), diff --git a/packages/stack/src/effect.integration.test.ts b/packages/stack/src/effect.integration.test.ts index c8be0f1e4d..722ed319a2 100644 --- a/packages/stack/src/effect.integration.test.ts +++ b/packages/stack/src/effect.integration.test.ts @@ -147,7 +147,7 @@ it.live("starts the owner on opt-in reopen without starting saved services", () ); it.live( - "reports a failed destroy and keeps the owner and saved data for retry", + "reports a failed destroy, exits the owner, and lets the next destroy reclaim the stack", () => Effect.scoped( Effect.gen(function* () { @@ -180,6 +180,7 @@ it.live( expect(running).toHaveLength(1); expect(running[0]?.host).toBeDefined(); + const released = yield* watchLeaseRelease(options.stateRoot, stack.id); const destroyExit = yield* stack.destroy.pipe(Effect.exit); expect(Exit.isFailure(destroyExit)).toBe(true); if (Exit.isFailure(destroyExit)) { @@ -197,9 +198,11 @@ it.live( }); } expect(yield* fs.readLink(dataRoot).pipe(Effect.isSuccess)).toBe(true); + yield* released; const retained = yield* discover(options); expect(retained).toHaveLength(1); - expect(retained[0]?.host?.pid).toBe(running[0]?.host?.pid); + expect(retained[0]?.host).toBeUndefined(); + yield* assertOwnerExited(running[0]?.host?.pid ?? 0); yield* fs.remove(dataRoot, { force: true }); yield* stack.destroy; @@ -209,6 +212,57 @@ it.live( { timeout: 30_000 }, ); +it.live( + "stops a running database within the drain deadline while a client holds an idle TCP connection", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-stop-idle-tcp-" }); + const options = { + projectRoot: root, + stateRoot: `${root}/state`, + cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + runtime: "native", + } satisfies Parameters[0]; + const stack = yield* create(options); + yield* Effect.addFinalizer(() => destroyTestStack(stack)); + const database = yield* stack.services.create({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("idle-tcp-password"), + jwtSecret: Redacted.make("idle-tcp-jwt-secret-at-least-thirty-two-characters"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* database.start; + yield* database.ready; + const { databaseUrl } = yield* database.credentials({ from: "runtime" }); + if (databaseUrl === undefined) return yield* Effect.die("database credentials missing"); + const url = new URL(databaseUrl); + const client = yield* Effect.callback((resume) => { + const connection = Net.createConnection({ + host: url.hostname, + port: Number(url.port), + }); + connection.once("connect", () => resume(Effect.succeed(connection))); + connection.once("error", (cause) => resume(Effect.fail(cause))); + return Effect.sync(() => connection.destroy()); + }); + + // The drain deadline is 10 seconds; an idle client must not make stop wait for it. + yield* stack.stop.pipe(Effect.timeout("5 seconds")); + + client.destroy(); + expect(yield* discover(options)).toHaveLength(1); + expect((yield* discover(options))[0]?.host).toBeUndefined(); + }), + ).pipe(Effect.provide(layer)), + { timeout: 60_000 }, +); + const resetDataStory = (runtime: "native" | "docker") => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -457,43 +511,83 @@ it.live("rejects an owner of another release while stop and destroy still reach }).pipe(Effect.scoped, Effect.provide(layer)), ); -it.live("treats a sweeper's hold as no owner and starts one once the sweep ends", () => +it.live( + "treats a sweeper's hold as no owner, refuses to stop, and starts one once the sweep ends", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-sweeping-" }); + const options = { + projectRoot: root, + stateRoot: `${root}/state`, + cacheRoot: `${root}/cache`, + runtime: "native", + } satisfies Parameters[0]; + const stack = yield* create(options); + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: options.stateRoot })), + ); + yield* Effect.ensuring( + Effect.gen(function* () { + const sweep = yield* Scope.make(); + const lease = yield* state.acquireLease(stack.id).pipe(Scope.provide(sweep)); + yield* lease.publishHolder({ + role: "sweeper", + pid: process.pid, + startedAt: "2026-01-01T00:00:00.000Z", + }); + const refused = yield* Effect.flip(stack.stop); + expect(refused.message).toContain("holds the stack lease"); + expect(yield* stack.composition.stop).toEqual([]); + expect((yield* discover(options))[0]?.host).toBeUndefined(); + + const starting = yield* stack.composition.start.pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* lease.retractHolder; + yield* Scope.close(sweep, Exit.void); + expect(yield* Fiber.join(starting)).toEqual([]); + expect((yield* discover(options))[0]?.host).toBeDefined(); + }), + destroyTestStack(stack), + ); + }).pipe(Effect.scoped, Effect.provide(layer)), +); + +it.live("reclaims a registered stack's leftovers when stopping without an owner", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-sweeping-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ownerless-stop-" }); const options = { projectRoot: root, stateRoot: `${root}/state`, cacheRoot: `${root}/cache`, runtime: "native", + lifetime: "session", } satisfies Parameters[0]; - const stack = yield* create(options); + const identity = yield* resolveStackIdentity(options); + const id = yield* deriveStackId(identity); const state = yield* StackNamespace.Service.pipe( Effect.provide(StackNamespace.layer({ root: options.stateRoot })), ); - yield* Effect.ensuring( - Effect.gen(function* () { - const sweep = yield* Scope.make(); - const lease = yield* state.acquireLease(stack.id).pipe(Scope.provide(sweep)); - yield* lease.publishHolder({ - role: "sweeper", - pid: process.pid, - startedAt: "2026-01-01T00:00:00.000Z", - }); - yield* stack.stop; - expect(yield* stack.composition.stop).toEqual([]); - expect((yield* discover(options))[0]?.host).toBeUndefined(); + yield* state.save({ + id, + identity, + lifetime: "session", + runtime: "native", + instances: [{ id: "leftover", creation: { service: "mail", config: {} } }], + composition: { members: [], dependencies: [] }, + ports: [], + }); + const leftover = `${options.stateRoot}/${id}/data/leftover`; + yield* fs.makeDirectory(leftover, { recursive: true }); + yield* fs.writeFileString(`${leftover}/state`, "left behind"); + const stack = yield* open({ ...options, id }); - const starting = yield* stack.composition.start.pipe( - Effect.forkChild({ startImmediately: true }), - ); - yield* lease.retractHolder; - yield* Scope.close(sweep, Exit.void); - expect(yield* Fiber.join(starting)).toEqual([]); - expect((yield* discover(options))[0]?.host).toBeDefined(); - }), - destroyTestStack(stack), - ); + yield* stack.stop; + + expect(yield* state.read(id)).toBeUndefined(); + expect(yield* fs.exists(leftover)).toBe(false); }).pipe(Effect.scoped, Effect.provide(layer)), ); diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index bfe977e438..d5b2258129 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -18,6 +18,7 @@ import { Stream, } from "effect"; import { HttpClient } from "effect/unstable/http"; +import { ChildProcessSpawner } from "effect/unstable/process"; import { RpcClientError } from "effect/unstable/rpc/RpcClientError"; import { connectHost, @@ -328,7 +329,12 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( // Calls and streams release what they borrow in their own scope, not in the handle's. const services = Context.omit(Scope.Scope)( yield* Effect.context< - HttpClient.HttpClient | FileSystem.FileSystem | Path.Path | Crypto.Crypto | Scope.Scope + | HttpClient.HttpClient + | FileSystem.FileSystem + | Path.Path + | Crypto.Crypto + | ChildProcessSpawner.ChildProcessSpawner + | Scope.Scope >(), ); const crypto = yield* Crypto.Crypto; @@ -475,6 +481,24 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( Effect.provideContext(services), ), ).pipe(Stream.mapError((cause) => failure(operation, cause))); + /** Stop with no owner: the leftovers of a failed or killed owner are reclaimed under the lease. */ + const reclaimWithoutOwner = reclaimStack({ + state, + stateRoot: locations.stateRoot, + cacheRoot: locations.cacheRoot, + id: saved.id, + }).pipe( + Effect.flatMap((reclaimed) => + reclaimed + ? Effect.void + : Effect.fail( + new StackError({ + operation: "shutdown", + message: "Another process holds the stack lease; run stop again", + }), + ), + ), + ); const shutdown = Effect.fn("Stack.shutdown")(function* (destroy: boolean) { const operation = destroy ? "destroy" : "shutdown"; yield* invalidate(); @@ -486,9 +510,11 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( const live = yield* connectHost(state, saved.id, { anyRelease: true }).pipe( Effect.map(Option.some), Effect.catchIf(ownerAbsent, () => - destroy - ? launchHost(state, launchOptions).pipe(Effect.map(Option.some)) - : Effect.succeed(Option.none()), + Effect.gen(function* () { + if (destroy) return Option.some(yield* launchHost(state, launchOptions)); + yield* reclaimWithoutOwner; + return Option.none(); + }), ), Effect.catchIf(stackGone, () => Effect.succeed(Option.none())), // The owner's startup sweep reports an unreachable engine before any owner serves. From 076c9ecf76040c9f3af9fd0b032ec6d528e81ad9 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 00:12:45 +0200 Subject: [PATCH 056/142] test(stack): relay docker service tests through the IPv4-only host alias --- packages/stack/tests/docker-relay.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/stack/tests/docker-relay.ts b/packages/stack/tests/docker-relay.ts index 003922d0a5..7490ae3123 100644 --- a/packages/stack/tests/docker-relay.ts +++ b/packages/stack/tests/docker-relay.ts @@ -1,6 +1,7 @@ import { Effect, Predicate } from "effect"; import { bindTcp, serveTcp, ProxyError } from "../src/Proxy.ts"; import { makeHttpProxy } from "../src/HttpProxy.ts"; +import { DOCKER_HOST_ALIAS } from "../src/runtime/Container.ts"; import type { ServiceEndpoint } from "../src/services/Recipe.ts"; const address = (endpoint: Effect.Effect) => @@ -22,7 +23,7 @@ export const makeDockerTcpRelay = Effect.fn("DockerRelay.makeTcp")( if (!Predicate.isTagged(listener.address, "TcpAddress")) return yield* Effect.die("Expected TCP relay listener"); yield* serveTcp(listener, address(endpoint), "docker-relay").pipe(Effect.forkScoped); - return { host: "host.docker.internal", port: listener.address.port }; + return { host: DOCKER_HOST_ALIAS, port: listener.address.port }; }), ); @@ -31,6 +32,6 @@ export const makeDockerHttpRelay = Effect.fn("DockerRelay.makeHttp")( Effect.gen(function* () { const proxy = yield* makeHttpProxy({ host: "0.0.0.0", port: 0 }); yield* proxy.setRoutes([{ id: "service", prefix: "/", target: address(endpoint) }]); - return { host: "host.docker.internal", port: proxy.port }; + return { host: DOCKER_HOST_ALIAS, port: proxy.port }; }), ); From 3eee00c85c6ed63aa91d20ce6c7da1d1ec3dcb53 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 00:18:39 +0200 Subject: [PATCH 057/142] fix(stack): reject HTTP work after drain begins and stop ownerless stacks from the CLI --- apps/cli/docs/stack-commands.md | 2 +- .../experimental/stack/stop/SIDE_EFFECTS.md | 18 +++-- .../experimental/stack/stop/stop.handler.ts | 28 ++++--- .../stack/stop/stop.integration.test.ts | 73 ++++++++++++++----- packages/stack/ARCHITECTURE.md | 3 +- .../stack/src/HttpProxy.integration.test.ts | 59 ++++++++++++++- packages/stack/src/HttpProxy.ts | 10 +++ .../stack/src/Network.integration.test.ts | 9 ++- 8 files changed, 155 insertions(+), 47 deletions(-) diff --git a/apps/cli/docs/stack-commands.md b/apps/cli/docs/stack-commands.md index e443c8d669..abb32853aa 100644 --- a/apps/cli/docs/stack-commands.md +++ b/apps/cli/docs/stack-commands.md @@ -299,7 +299,7 @@ retained in stack state, so starting without `--exclude` restores the project's cannot be read or decoded are skipped with a warning on stderr; only a failure to read the stacks directory itself fails discovery, before any stop is attempted. Individual stop failures make the command fail and identify the affected stack IDs with their error details; no -success or unavailable summary is emitted when a stop fails. +success or not-running summary is emitted when a stop fails. `supabase stack destroy --stack feature-a` permanently removes exactly that stack and its data after confirmation. Use `--yes` for unattended execution. There is no bulk destroy option. diff --git a/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md index 1bfd6e22e9..02418df95c 100644 --- a/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md @@ -12,9 +12,11 @@ with either selector. Explicit legacy `-o/--output` is rejected; use `--output-format` instead. Discovery probes saved owners before shutdown. If no owner is reachable, the -command reports `No owner is reachable; workload state is unavailable.` It -neither starts an owner nor claims workloads have stopped. If an owner disappears -after that preflight, the shutdown error remains a failure. +command does not start one; it reclaims the stack's leftover workloads under the +stack lease and reports `Stack was not running; leftover resources were reclaimed.` +It fails when another process holds the lease or the cleanup fails, so a stack +whose owner exited after a failed stop can be stopped again. If an owner +disappears after discovery, the shutdown error remains a failure. Before any service stops, the owner stops accepting new connections on every stack listener and lets connections already established keep flowing for up to @@ -22,12 +24,12 @@ stack listener and lets connections already established keep flowing for up to issued can still complete, at the cost of the command waiting up to that long longer for shutdown to finish. -Text confirms each successful shutdown and identifies each unavailable owner. -JSON and stream-json success data contain `stopped` and `unavailable` ID arrays. +Text confirms each successful shutdown and identifies each stack that had no owner. +JSON and stream-json success data contain `stopped` and `notRunning` ID arrays. A missing default selection reports `found: false`; an unknown explicit name or ID fails. A single selection reads only the selected stack's state document; an unreadable document fails the selection instead of being reported as missing. -`--all` attempts every selected reachable owner and reports failures with their +`--all` attempts every selected stack and reports failures with their IDs. It skips state entries that cannot be read or decoded with a warning on stderr identifying each stack; only a failure to read the stacks directory itself fails discovery. @@ -45,8 +47,8 @@ same way. No caller-owned upload files are removed. ## Exit codes and telemetry -Exit 0 on successful shutdown, absent default selection, or unavailable owner; -1 for invalid flags, selection/registry failure, or shutdown failure; 130 on +Exit 0 on successful shutdown or reclaim, or absent default selection; +1 for invalid flags, selection/registry failure, or shutdown or reclaim failure; 130 on interruption. The command wrapper retains standard command telemetry, with no custom events. Telemetry flushes on success and failure to `/telemetry.json`. diff --git a/apps/cli/src/commands/experimental/stack/stop/stop.handler.ts b/apps/cli/src/commands/experimental/stack/stop/stop.handler.ts index 6dd789617f..df8858cb7a 100644 --- a/apps/cli/src/commands/experimental/stack/stop/stop.handler.ts +++ b/apps/cli/src/commands/experimental/stack/stop/stop.handler.ts @@ -119,17 +119,15 @@ export const stackStop = Effect.fn("experimental.stack.stop")(function* (flags: hostRunning, }): Effect.Effect<{ readonly id: string; - readonly result: Result.Result<"stopped" | "unavailable", StackError>; + readonly result: Result.Result<"stopped" | "not-running", StackError>; }> => - !hostRunning - ? Effect.succeed({ id, result: Result.succeed("unavailable" as const) }) - : api.open({ ...locations, id }).pipe( - Effect.flatMap((stack) => stack.stop), - Effect.scoped, - Effect.as("stopped" as const), - Effect.result, - Effect.map((result) => ({ id, result })), - ), + api.open({ ...locations, id }).pipe( + Effect.flatMap((stack) => stack.stop), + Effect.scoped, + Effect.as(hostRunning ? ("stopped" as const) : ("not-running" as const)), + Effect.result, + Effect.map((result) => ({ id, result })), + ), ); const failures = results.flatMap(({ id, result }) => Result.isFailure(result) ? [{ id, error: result.failure }] : [], @@ -144,8 +142,8 @@ export const stackStop = Effect.fn("experimental.stack.stop")(function* (flags: const stopped = results.flatMap(({ id, result }) => Result.isSuccess(result) && result.success === "stopped" ? [id] : [], ); - const unavailable = results.flatMap(({ id, result }) => - Result.isSuccess(result) && result.success === "unavailable" ? [id] : [], + const notRunning = results.flatMap(({ id, result }) => + Result.isSuccess(result) && result.success === "not-running" ? [id] : [], ); yield* Effect.annotateCurrentSpan({ "stack.count": results.length, @@ -153,9 +151,9 @@ export const stackStop = Effect.fn("experimental.stack.stop")(function* (flags: }); if (output.format === "text") { for (const id of stopped) yield* output.raw(`Stack ${id} stopped.\n`); - for (const id of unavailable) - yield* output.raw(`Stack ${id}: No owner is reachable; workload state is unavailable.\n`); + for (const id of notRunning) + yield* output.raw(`Stack ${id} was not running; leftover resources were reclaimed.\n`); if (results.length === 0) yield* output.raw("No managed stacks found.\n"); - } else yield* output.success("", { stopped, unavailable }); + } else yield* output.success("", { stopped, notRunning }); }).pipe(Effect.ensuring(telemetry.flush)); }); diff --git a/apps/cli/src/commands/experimental/stack/stop/stop.integration.test.ts b/apps/cli/src/commands/experimental/stack/stop/stop.integration.test.ts index 6648036cf2..7fd6437ec8 100644 --- a/apps/cli/src/commands/experimental/stack/stop/stop.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stop/stop.integration.test.ts @@ -1,12 +1,13 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Effect, FileSystem, Layer, Option, Path, Ref } from "effect"; +import { Effect, Exit, FileSystem, Layer, Option, Path, Ref, Scope } from "effect"; import { StackError } from "@supabase/stack/effect"; import { mockOutput } from "../../../../../tests/helpers/mocks.ts"; import { mockCommandSettings, mockTelemetryStateTracked, } from "../../../../../tests/helpers/command-mocks.ts"; +import * as StackNamespace from "../../../../../../../packages/stack/src/StackNamespace.ts"; import { StackApi, stackApiLayer, stackTargetResolverLayer } from "../stack.shared.ts"; import { stackStop } from "./stop.handler.ts"; @@ -56,24 +57,28 @@ describe("stack stop", () => { }).pipe(Effect.provide(live)), ); - it.live("stop all preserves an offline registry without claiming stopped workloads", () => - Effect.gen(function* () { - const f = yield* fixture(); - yield* f.api.create({ - ...f.locations, - projectRoot: f.root, - name: "second", - runtime: "native", - }); - yield* stackStop({ ...flags(), all: Option.some(true) }).pipe(Effect.provide(f.layer)); - yield* stackStop({ ...flags(), all: Option.some(true) }).pipe(Effect.provide(f.layer)); - expect(f.output.stdoutText.match(/workload state is unavailable/g)).toHaveLength(4); - expect(f.output.stdoutText).not.toContain("stopped."); - expect((yield* f.api.discover(f.locations)).every(({ host }) => host === undefined)).toBe( - true, - ); - expect(f.telemetry.flushed).toBe(true); - }).pipe(Effect.provide(live)), + it.live( + "stop all reclaims ownerless stacks and keeps their definitions without claiming a stop", + () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* f.api.create({ + ...f.locations, + projectRoot: f.root, + name: "second", + runtime: "native", + }); + yield* stackStop({ ...flags(), all: Option.some(true) }).pipe(Effect.provide(f.layer)); + yield* stackStop({ ...flags(), all: Option.some(true) }).pipe(Effect.provide(f.layer)); + expect( + f.output.stdoutText.match(/was not running; leftover resources were reclaimed/g), + ).toHaveLength(4); + expect(f.output.stdoutText).not.toContain("stopped."); + expect((yield* f.api.discover(f.locations)).every(({ host }) => host === undefined)).toBe( + true, + ); + expect(f.telemetry.flushed).toBe(true); + }).pipe(Effect.provide(live)), ); it.live.skipIf(process.platform === "win32" || process.getuid?.() === 0)( @@ -92,7 +97,7 @@ describe("stack stop", () => { fs.chmod(locked, 0o700).pipe(Effect.orDie), ); yield* stackStop({ ...flags(), all: Option.some(true) }).pipe(Effect.provide(f.layer)); - expect(f.output.stdoutText.match(/workload state is unavailable/g)).toHaveLength(1); + expect(f.output.stdoutText.match(/was not running/g)).toHaveLength(1); expect(f.output.stderrText).toContain("Warning: skipping invalid stack broken"); expect(f.output.stderrText).toContain("Warning: skipping invalid stack locked"); expect(f.output.stdoutText).not.toContain("Warning"); @@ -142,4 +147,32 @@ describe("stack stop", () => { expect(f.telemetry.flushed).toBe(true); }).pipe(Effect.provide(live)), ); + + it.live("fails when an ownerless stack's lease is held by another process", () => + Effect.gen(function* () { + const f = yield* fixture(); + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: f.locations.stateRoot })), + ); + const holder = yield* Scope.make(); + const lease = yield* state.acquireLease(f.stack.id).pipe(Scope.provide(holder)); + yield* lease.publishHolder({ + role: "sweeper", + pid: process.pid, + startedAt: "2026-01-01T00:00:00.000Z", + }); + const error = yield* stackStop(flags(f.stack.id)).pipe( + Effect.provide(f.layer), + Effect.flip, + Effect.ensuring(Scope.close(holder, Exit.void)), + ); + expect(error.reason).toBe("unknown"); + expect(error.detail).toContain(`${f.stack.id}: Another process holds the stack lease`); + expect(f.output.stdoutText).toBe(""); + expect((yield* f.api.discover(f.locations)).map(({ definition }) => definition.id)).toEqual([ + f.stack.id, + ]); + expect(f.telemetry.flushed).toBe(true); + }).pipe(Effect.provide(live)), + ); }); diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 3606aec693..7fa99eeee8 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -523,7 +523,8 @@ During Draining: **Connection drain.** Stop and destroy share one drain step, run before any service stops: accept closes on every stack listener, public and internal dependency traffic alike (they are the same -listeners), so no new connection is admitted. Drain then waits, for at most 10 seconds, only for +listeners), so no new connection is admitted, and a request or upgrade arriving on an established HTTP +connection is refused (503 with `Connection: close`, or a destroyed socket). Drain then waits, for at most 10 seconds, only for observable work: HTTP requests in flight and upgraded sockets such as WebSockets. TCP connections never count, so an idle pooled connection (PostgREST's database pool, for example) cannot delay shutdown. Services then stop, and listener scopes close afterward through the ordinary diff --git a/packages/stack/src/HttpProxy.integration.test.ts b/packages/stack/src/HttpProxy.integration.test.ts index 458993ef50..e6bb9de550 100644 --- a/packages/stack/src/HttpProxy.integration.test.ts +++ b/packages/stack/src/HttpProxy.integration.test.ts @@ -1,6 +1,6 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Data, Deferred, Effect, Fiber, Layer, SubscriptionRef } from "effect"; +import { Data, Deferred, Effect, Fiber, Layer, Stream, SubscriptionRef } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { createServer, type Server, type ServerResponse } from "node:http"; // oxlint-disable-line effecttsgo/node-builtin-import -- raw server fixture. import { createServer as createTcpServer, Socket, type Server as NetServer } from "node:net"; // oxlint-disable-line effecttsgo/node-builtin-import -- raw disconnect fixture. @@ -1242,3 +1242,60 @@ it.live( }), ).pipe(Effect.provide(NodeServices.layer)), ); + +it.live( + "answers a request submitted on an established connection after stopAccepting with 503 while the in-flight response completes", + () => + Effect.scoped( + Effect.gen(function* () { + let received = 0; + const backend = createServer((_incoming, outgoing) => { + received += 1; + outgoing.writeHead(200, { "content-length": "9" }); + // Held open: the test finishes this response after the late request has been sent. + }); + const backendAddress = yield* listen(backend); + const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); + yield* proxy.setRoutes([ + { id: "api", prefix: "/", target: Effect.succeed(backendAddress) }, + ]); + + const heldResponseFiber = yield* Effect.callback( + (resume) => { + const onRequest = (_incoming: unknown, outgoing: ServerResponse) => + resume(Effect.succeed(outgoing)); + backend.once("request", onRequest); + return Effect.sync(() => backend.off("request", onRequest)); + }, + ).pipe(Effect.forkScoped); + const client = yield* connectRaw(proxy.port, proxy.host); + const closed = yield* Effect.callback((resume) => { + const chunks: Array = []; + client.on("data", (chunk: Buffer) => chunks.push(chunk)); + client.once("close", () => resume(Effect.succeed(Buffer.concat(chunks).toString()))); + return Effect.void; + }).pipe(Effect.timeout("5 seconds"), Effect.forkScoped); + client.write("GET /slow HTTP/1.1\r\nHost: localhost\r\nConnection: keep-alive\r\n\r\n"); + const heldResponse = yield* Fiber.join(heldResponseFiber); + expect(yield* SubscriptionRef.get(proxy.outstandingConnections)).toBe(1); + + yield* proxy.stopAccepting; + client.write("GET /late HTTP/1.1\r\nHost: localhost\r\nConnection: keep-alive\r\n\r\n"); + heldResponse.end("slow-body"); + + const transcript = yield* Fiber.join(closed); + const inFlight = transcript.indexOf("HTTP/1.1 200"); + const late = transcript.indexOf("HTTP/1.1 503"); + expect(inFlight).toBeGreaterThanOrEqual(0); + expect(late).toBeGreaterThan(inFlight); + expect(transcript.slice(inFlight, late)).toContain("slow-body"); + expect(transcript.slice(late).toLowerCase()).toContain("connection: close"); + expect(received).toBe(1); + yield* SubscriptionRef.changes(proxy.outstandingConnections).pipe( + Stream.filter((count) => count === 0), + Stream.runHead, + Effect.timeout("5 seconds"), + ); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index ca5bc61d73..d45f22c157 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -438,6 +438,12 @@ export const makeHttpProxy = (options: { if (!accepting) server.closeIdleConnections(); }; const server = createServer((request, response) => { + if (!accepting) { + response.statusCode = 503; + response.setHeader("Connection", "close"); + response.end("Service Unavailable"); + return; + } Effect.runSyncWith(services)( SubscriptionRef.update(outstandingConnections, (count) => count + 1), ); @@ -496,6 +502,10 @@ export const makeHttpProxy = (options: { }); server.on("upgrade", (request, socket, head) => { socket.on("error", () => socket.destroy()); + if (!accepting) { + socket.destroy(); + return; + } Effect.runSyncWith(services)( SubscriptionRef.update(outstandingConnections, (count) => count + 1), ); diff --git a/packages/stack/src/Network.integration.test.ts b/packages/stack/src/Network.integration.test.ts index 8e29f0f067..edf144b9de 100644 --- a/packages/stack/src/Network.integration.test.ts +++ b/packages/stack/src/Network.integration.test.ts @@ -693,9 +693,16 @@ it.live("drains past an idle TCP connection without ever reaching the deadline", }); yield* service.bind; const address = yield* service.address("sql", "host"); + // A keep-alive round trip through the TCP endpoint proves the listener accepted and tracks + // this connection; the socket then sits idle through drain. const socket = yield* Effect.callback((resume) => { const connection = Net.createConnection({ host: address.host, port: address.port }); - connection.once("connect", () => resume(Effect.succeed(connection))); + connection.once("connect", () => + connection.write( + `GET /hello HTTP/1.1\r\nHost: ${address.host}:${address.port}\r\nConnection: keep-alive\r\n\r\n`, + ), + ); + connection.once("data", () => resume(Effect.succeed(connection))); connection.on("error", (cause) => resume(Effect.fail(new FixtureError({ message: cause.message }))), ); From 88627e5efb2935e73655c7faf1c6ac7fd0c3fffa Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 00:32:34 +0200 Subject: [PATCH 058/142] refactor(stack): sweep artifact generations only on cache misses and drop the in-process lock guard Retirement and stale-staging reaping now run only when a request misses the cache, after the new generation is published; cache hits do no sweep. SQLite tracks locks per file across every connection in a process, and the stack opens lock files only through node:sqlite, so a second in-process connection already contends like another process. That makes the open-paths guard, the pin reference-count registry, the local-pin bypass for retirement and the shared connection cache redundant; each pin now owns its connection and the port registry's connection is scoped to its layer. --- packages/stack/ARCHITECTURE.md | 3 + packages/stack/README.md | 2 +- .../src/namespace/Pin.integration.test.ts | 67 ++++++++++++ packages/stack/src/namespace/Pin.ts | 100 +++-------------- .../stack/src/namespace/PortReservations.ts | 10 +- .../drivers/Sqlite.integration.test.ts | 28 +++++ .../stack/src/namespace/drivers/Sqlite.ts | 67 ++++-------- .../stack/src/preparation/ArtifactStore.ts | 42 ++++---- .../preparation/artifacts.integration.test.ts | 101 +++++++++++++----- 9 files changed, 235 insertions(+), 185 deletions(-) create mode 100644 packages/stack/src/namespace/Pin.integration.test.ts create mode 100644 packages/stack/src/namespace/drivers/Sqlite.integration.test.ts diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 7fa99eeee8..06cd7f8585 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -688,6 +688,9 @@ process death also releases ownership. The file stays in place and is accessed o SQLite. No tables, state records, or WAL are created there. Saved stack data remains in JSON; the lock does not make multi-file operations transactional or recover interrupted operations. This uses the built-in SQLite API available in the pinned Bun runtime and modern Node.js. +Every lock connection in a process goes through that one SQLite library, which tracks locks per +file across connections, so a second connection in the same process is excluded exactly as another +process would be, and closing one connection never releases a lock another still holds. The artifact cache is independent and shared across stacks. Normal stop preserves the stack directory and service data. Destroy removes the state document, owner files and proven-owned, empty parents; the lease file goes last, while its lock is still held; caller-owned paths such as Storage uploads remain untouched. diff --git a/packages/stack/README.md b/packages/stack/README.md index 6b7cb62f3a..3248d8334a 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -85,7 +85,7 @@ Omitted database `jwtSecret` and `rootKey` inputs use the shared local-developme as `DEFAULT_LOCAL_JWT_SECRET` and `DEFAULT_POSTGRES_ROOT_KEY`. Explicit values override these defaults. The effective root key is supplied through a stack-owned file for both native and container runtimes. -The native artifact cache holds one immutable, content-addressed generation per published archive at `cacheRoot///`, published by renaming a lock-guarded `cacheRoot//.staging//content` into place. A generation is never modified after publish. `Artifacts.use` is the one scoped launch-time operation: it takes a SHARED lock on the generation's `cacheRoot//.lock` file before resolving or preparing it, and every consumer (service launch, startup commands, the PostgreSQL handover, one-shot commands, and the native launcher itself) uses the returned paths only inside that scope. A background sweep at prepare time retires a generation only once it is both unlocked and untouched for 30 days; digest lock files are never deleted. A stale, lock-guarded staging directory left by a killed preparer is reaped by the next preparer or sweep, never by age. +The native artifact cache holds one immutable, content-addressed generation per published archive at `cacheRoot///`, published by renaming a lock-guarded `cacheRoot//.staging//content` into place. A generation is never modified after publish. `Artifacts.use` is the one scoped launch-time operation: it takes a SHARED lock on the generation's `cacheRoot//.lock` file before resolving or preparing it, and every consumer (service launch, startup commands, the PostgreSQL handover, one-shot commands, and the native launcher itself) uses the returned paths only inside that scope. Each pin holds its own connection, and the SQLite lock a pin takes excludes retirement whether the pin belongs to this process or another. After a cache miss publishes a generation, a sweep over the whole cache root retires a generation only once it is both unlocked and untouched for 30 days; cache hits never sweep, and digest lock files are never deleted. A stale, lock-guarded staging directory left by a killed preparer is reaped by the next cache miss, never by age. Native PostgreSQL refuses to run as uid 0. When the stack runs as root inside a detected agent sandbox (Claude Code, Modal Sandbox), or `SUPABASE_NATIVE_POSTGRES_USER=` names a non-root system user, only the PostgreSQL process runs as that user: the instance data directory, root key file, socket directory, and the bundle's `pgsodium_getkey.sh` are chowned to it, and the instance directory and its ancestors outside the generation (the artifact cache and stack state roots, which otherwise stay restricted to their owner) receive traverse-only (`o+x`) permission. A directory inside the generation itself is never chmodded; it must already carry that bit from the archive, or the handover fails naming the path. The `pgsodium_getkey.sh` chown is the one accepted exception to generation immutability: the packaged init script hard-codes that cached path and `chmod +x`s it, which only its owner may do. Running as root elsewhere fails before PostgreSQL launches. diff --git a/packages/stack/src/namespace/Pin.integration.test.ts b/packages/stack/src/namespace/Pin.integration.test.ts new file mode 100644 index 0000000000..683c42cd13 --- /dev/null +++ b/packages/stack/src/namespace/Pin.integration.test.ts @@ -0,0 +1,67 @@ +import { NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit, FileSystem, Scope, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { pin } from "./Pin.ts"; + +/** Reports whether a separate process can take an EXCLUSIVE lock on `lockPath` right now. */ +const exclusiveLockOutcome = (lockPath: string) => + Effect.scoped( + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const script = ` + import { DatabaseSync } from "node:sqlite"; + const database = new DatabaseSync(process.argv[1]); + database.exec("PRAGMA busy_timeout = 0"); + try { + database.exec("BEGIN EXCLUSIVE"); + console.log("acquired"); + } catch (error) { + console.log("busy"); + } + `; + const child = yield* spawner.spawn( + ChildProcess.make(process.execPath, ["-e", script, lockPath], { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }), + ); + const [stdout, stderr, exitCode] = yield* Effect.all( + [ + child.stdout.pipe(Stream.decodeText, Stream.mkString), + child.stderr.pipe(Stream.decodeText, Stream.mkString), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + expect(Number(exitCode), stderr).toBe(0); + return stdout.trim(); + }), + ); + +describe("pins", () => { + it.live( + "another process stays excluded after a second in-process pin on the same file is released", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-pin-" }); + const lockPath = `${directory}/generation.lock`; + + const holderScope = yield* Scope.make(); + yield* pin(lockPath).pipe(Scope.provide(holderScope)); + const competitorScope = yield* Scope.make(); + yield* pin(lockPath).pipe(Scope.provide(competitorScope)); + yield* Scope.close(competitorScope, Exit.void); + + expect(yield* exclusiveLockOutcome(lockPath)).toBe("busy"); + + yield* Scope.close(holderScope, Exit.void); + expect(yield* exclusiveLockOutcome(lockPath)).toBe("acquired"); + }), + ).pipe(Effect.provide(NodeServices.layer)), + 20_000, + ); +}); diff --git a/packages/stack/src/namespace/Pin.ts b/packages/stack/src/namespace/Pin.ts index 1461de46ee..cb8af6472e 100644 --- a/packages/stack/src/namespace/Pin.ts +++ b/packages/stack/src/namespace/Pin.ts @@ -1,32 +1,7 @@ -import { DateTime, Effect, Exit, FileSystem, Option, Schedule, Scope, Semaphore } from "effect"; +import { DateTime, Effect, FileSystem, Schedule, Scope } from "effect"; import { namespaceError, type NamespaceError } from "./Capabilities.ts"; import { acquireLock, isBusy, takeSharedLock } from "./drivers/Sqlite.ts"; -/** - * Open reader connections this process currently holds, keyed by canonical digest lock path. A - * second `acquireLock` on the same path would either be rejected by its own single-holder guard or - * (worse) silently coalesce, since POSIX advisory locks belong to a process, not a descriptor. So - * every local pin on the same generation shares this one connection, reference-counted, instead of - * opening a second one; this is also the registry a retirement sweep's own `acquireLock` attempt - * naturally defers to, since that attempt targets the same path and the same single-holder guard. - */ -const registry = new Map(); - -/** - * Serializes the check-or-create section per path: checking `registry` and opening its connection - * span several `yield*` suspension points, so two concurrent local pins on a brand new path would - * otherwise both see it missing and race to open it. One semaphore per path, created and looked up - * synchronously (never itself awaited to exist), rules that out. - */ -const locks = new Map(); -const semaphoreFor = (path: string): Semaphore.Semaphore => { - const existing = locks.get(path); - if (existing !== undefined) return existing; - const created = Semaphore.makeUnsafe(1); - locks.set(path, created); - return created; -}; - /** Rides out a retirement sweep's brief EXCLUSIVE hold instead of failing a concurrent pin. */ const pinRetrySchedule = Schedule.spaced("10 millis").pipe( Schedule.upTo({ duration: "2 seconds" }), @@ -34,66 +9,21 @@ const pinRetrySchedule = Schedule.spaced("10 millis").pipe( /** * Holds a SHARED lock on `lockPath` (one generation's digest lock file) for the life of the - * returned scope: every consumer of a prepared generation's paths must stay inside this scope, and + * enclosing scope: every consumer of a prepared generation's paths must stay inside this scope, and * must not resolve or create `lockPath`'s generation directory before this resolves. Touches the - * lock file's mtime on first acquisition in this process, which is retirement's only liveness - * signal for a holder in another process. The caller must ensure `lockPath`'s parent directory - * already exists. - */ -export const pin = ( - lockPath: string, -): Effect.Effect => - Effect.acquireRelease( - semaphoreFor(lockPath).withPermits(1)( - Effect.gen(function* () { - const existing = registry.get(lockPath); - if (existing !== undefined) { - existing.refCount++; - return; - } - const scope = yield* Scope.make(); - yield* Effect.gen(function* () { - const connection = yield* acquireLock(lockPath, "create").pipe(Scope.provide(scope)); - yield* takeSharedLock(connection).pipe( - Effect.retry({ schedule: pinRetrySchedule, while: isBusy }), - ); - const fs = yield* FileSystem.FileSystem; - const now = DateTime.toDateUtc(yield* DateTime.now); - yield* fs - .utimes(lockPath, now, now) - .pipe(Effect.mapError((cause) => namespaceError("pin-touch", cause))); - }).pipe(Effect.tapError(() => Scope.close(scope, Exit.void))); - registry.set(lockPath, { scope, refCount: 1 }); - }), - ), - () => - semaphoreFor(lockPath).withPermits(1)( - Effect.suspend(() => { - const entry = registry.get(lockPath); - if (entry === undefined) return Effect.void; - entry.refCount--; - if (entry.refCount > 0) return Effect.void; - registry.delete(lockPath); - return Scope.close(entry.scope, Exit.void); - }), - ), - ); - -/** - * Runs `attempt` (a non-pinning, best-effort reader like a retirement sweep) only while this - * process holds no live local pin on `lockPath`, under the same per-path permit `pin` itself - * takes. Without this, a sweep's own `acquireLock` and a concurrent first-time `pin` on the same - * path race to open independent connections, and the loser sees "locked by this process" instead - * of either properly queuing or deferring; sharing the permit makes them mutually exclusive, so a - * sweep either runs to completion before a fresh pin ever opens a connection, or correctly finds - * the path already pinned and skips. + * lock file's mtime on acquisition, which is retirement's only liveness signal for a holder in + * another process. The caller must ensure `lockPath`'s parent directory already exists. */ -export const withoutLocalPin = ( +export const pin = Effect.fn("Pin.pin")(function* ( lockPath: string, - attempt: Effect.Effect, -): Effect.Effect, E, R> => - semaphoreFor(lockPath).withPermits(1)( - Effect.suspend((): Effect.Effect, E, R> => - registry.has(lockPath) ? Effect.succeed(Option.none()) : Effect.map(attempt, Option.some), - ), +): Effect.fn.Return { + const fs = yield* FileSystem.FileSystem; + const connection = yield* acquireLock(lockPath, "create"); + yield* takeSharedLock(connection).pipe( + Effect.retry({ schedule: pinRetrySchedule, while: isBusy }), ); + const now = DateTime.toDateUtc(yield* DateTime.now); + yield* fs + .utimes(lockPath, now, now) + .pipe(Effect.mapError((cause) => namespaceError("pin-touch", cause))); +}); diff --git a/packages/stack/src/namespace/PortReservations.ts b/packages/stack/src/namespace/PortReservations.ts index 6eb2f121ae..8919061c8d 100644 --- a/packages/stack/src/namespace/PortReservations.ts +++ b/packages/stack/src/namespace/PortReservations.ts @@ -1,10 +1,10 @@ import { userInfo } from "node:os"; import type { DatabaseSync } from "node:sqlite"; import { NodeServices } from "@effect/platform-node"; -import { Context, Effect, FileSystem, Layer, Path, Predicate, Stream } from "effect"; +import { Context, Effect, FileSystem, Layer, Path, Predicate, Scope, Stream } from "effect"; import { ChildProcess } from "effect/unstable/process"; import { namespaceError, type NamespaceError } from "./Capabilities.ts"; -import { errcode, openSharedConnection } from "./drivers/Sqlite.ts"; +import { errcode, openDatabase } from "./drivers/Sqlite.ts"; import { lstatPath } from "./drivers/FileSystem.ts"; /** Identifies the stack that owns a port reservation. */ @@ -88,8 +88,8 @@ const isHolderRow = (value: unknown): value is HolderRow => Predicate.hasProperty(value, "stack_id") && typeof value.stack_id === "string"; -const connectionFor = (file: string): Effect.Effect => - openSharedConnection(file, (connection) => { +const connectionFor = (file: string): Effect.Effect => + openDatabase(file, (connection) => { // Short and bounded: real contention between two of this user's processes is brief, and // a blocking wait here is simpler than an async retry loop for a database this small. Set // before the schema statement, so even first-time table creation waits out a racing peer @@ -205,7 +205,7 @@ const resolvedHome: Effect.Effect = Effect.runSync( const make = Effect.fn("PortReservations.make")(function* (): Effect.fn.Return< Interface, NamespaceError, - FileSystem.FileSystem | Path.Path + FileSystem.FileSystem | Path.Path | Scope.Scope > { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; diff --git a/packages/stack/src/namespace/drivers/Sqlite.integration.test.ts b/packages/stack/src/namespace/drivers/Sqlite.integration.test.ts new file mode 100644 index 0000000000..51102a7afb --- /dev/null +++ b/packages/stack/src/namespace/drivers/Sqlite.integration.test.ts @@ -0,0 +1,28 @@ +import { NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit, FileSystem, Scope } from "effect"; +import { acquireLock, isBusy, takeLock } from "./Sqlite.ts"; + +const lockHolder = (lockPath: string, mode: "create" | "existing") => + acquireLock(lockPath, mode).pipe(Effect.flatMap(takeLock)); + +describe("SQLite lock driver", () => { + it.live("two holders in one process exclude each other until the first releases", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-lock-" }); + const lockPath = `${directory}/holder.lock`; + + const firstScope = yield* Scope.make(); + yield* lockHolder(lockPath, "create").pipe(Scope.provide(firstScope)); + + const refused = yield* Effect.scoped(lockHolder(lockPath, "existing")).pipe(Effect.flip); + expect(isBusy(refused)).toBe(true); + + yield* Scope.close(firstScope, Exit.void); + yield* Effect.scoped(lockHolder(lockPath, "existing")); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); +}); diff --git a/packages/stack/src/namespace/drivers/Sqlite.ts b/packages/stack/src/namespace/drivers/Sqlite.ts index 86544c7a1c..2c062b0281 100644 --- a/packages/stack/src/namespace/drivers/Sqlite.ts +++ b/packages/stack/src/namespace/drivers/Sqlite.ts @@ -30,23 +30,15 @@ const closeConnection = (connection: DatabaseSync) => }); /** - * One connection per resolved database path for a caller's process lifetime, running `init` only - * on first open. POSIX advisory locks (and the SQLite locks built on them) belong to a process, - * not a descriptor: a second connection this process opened on the same path would not see the - * first one's transaction as contention, so every caller sharing a path must share one connection - * instead of risking that coalescing. + * Opens `path` for the enclosing scope and runs `init` once on the new connection; a failing + * `init` closes the connection before the error propagates. */ -const sharedConnections = new Map(); - -/** Opens (or reuses) `path`'s process-lifetime shared connection; never closed until the process exits. */ -export const openSharedConnection = ( +export const openDatabase = ( path: string, init: (connection: DatabaseSync) => void, -): Effect.Effect => - Effect.suspend(() => { - const existing = sharedConnections.get(path); - if (existing !== undefined) return Effect.succeed(existing); - return Effect.try({ +): Effect.Effect => + Effect.acquireRelease( + Effect.try({ try: () => { const connection = new DatabaseSync(path); try { @@ -55,12 +47,15 @@ export const openSharedConnection = ( connection.close(); throw error; } - sharedConnections.set(path, connection); return connection; }, catch: (cause) => namespaceError("open", cause), - }); - }); + }), + (connection) => + closeConnection(connection).pipe( + Effect.catch((error) => Effect.logWarning(`Unable to close ${path}`, error)), + ), + ); /** Matches a `NamespaceError`'s cause against one of `node:sqlite`'s numeric `errcode`s. */ export const errcode = (cause: unknown, code: number) => @@ -69,41 +64,21 @@ export const errcode = (cause: unknown, code: number) => export const isBusy = (error: NamespaceError) => errcode(error.cause, 5); /** `acquireLock("existing")` reports this errcode when the database file does not exist. */ export const isMissing = (error: NamespaceError) => errcode(error.cause, 14); -const busy = (operation: string) => - namespaceError(operation, Object.assign(new Error("locked by this process"), { errcode: 5 })); - /** - * Canonical paths this process currently has a connection open on. POSIX advisory locks (and the - * SQLite locks built on them) belong to a process, not a descriptor: a second connection this - * process opened on the same path would not see the first one's lock as contention, so it would - * wrongly appear to succeed instead of reporting busy. Checking this set first, before opening a - * second descriptor, rules that out: a path already open in this process is always busy. - */ -const openPaths = new Set(); - -/** - * Opens `path`'s connection for the enclosing scope; busy if this process already has it open. - * Reserving the path and opening the connection happen as one `acquireRelease` acquisition, so an - * interruption can never leave the path reserved without a connection whose release clears it. + * Opens `path`'s connection for the enclosing scope. Every connection this process opens on a + * path shares one SQLite library instance, which tracks locks per inode across connections and + * keeps them held until the last one closes, so a second in-process connection contends with the + * first exactly as another process would. */ export const acquireLock = ( path: string, mode: "create" | "existing", ): Effect.Effect => - Effect.gen(function* () { - if (openPaths.has(path)) return yield* busy("lock"); - return yield* Effect.acquireRelease( - Effect.sync(() => openPaths.add(path)).pipe( - Effect.andThen(openConnection(path, mode)), - Effect.tapError(() => Effect.sync(() => openPaths.delete(path))), - ), - (connection) => - closeConnection(connection).pipe( - Effect.catch((error) => Effect.logWarning(`Unable to close ${path}`, error)), - Effect.ensuring(Effect.sync(() => openPaths.delete(path))), - ), - ); - }); + Effect.acquireRelease(openConnection(path, mode), (connection) => + closeConnection(connection).pipe( + Effect.catch((error) => Effect.logWarning(`Unable to close ${path}`, error)), + ), + ); /** * Takes the connection's RESERVED write lock once; fails busy under any contention instead of diff --git a/packages/stack/src/preparation/ArtifactStore.ts b/packages/stack/src/preparation/ArtifactStore.ts index 08e6822faf..61cdc70ee8 100644 --- a/packages/stack/src/preparation/ArtifactStore.ts +++ b/packages/stack/src/preparation/ArtifactStore.ts @@ -632,9 +632,8 @@ const discoverGenerations = ( * AND, rechecked while holding that lock, its mtime is older than the retention window. Digest * lock files are never deleted (a stable inode); only the generation directory they guard is. A * per-digest failure (including contention from a live pin or another retirer) is skipped, never - * propagated: this sweep must never block the `prepare`/`use` call it runs alongside. Each - * digest's own lock attempt shares `Pin`'s per-path permit for that digest, so it can never open a - * second, independent connection alongside a concurrent first-time local `pin` on the same path. + * propagated: this sweep must never block the `prepare`/`use` call it runs alongside. A pin held + * by this process contends with the sweep's lock exactly as a pin held by another process does. */ const retireStaleGenerations = Effect.fn("ArtifactStore.retire")(function* ( fs: FileSystem.FileSystem, @@ -648,24 +647,18 @@ const retireStaleGenerations = Effect.fn("ArtifactStore.retire")(function* ( const generationPath = path.join(keyRoot, digest); const lockPath = path.join(keyRoot, `${digest}.lock`); const stagingRoot = path.join(keyRoot, STAGING_DIR_NAME); - const didRetire = yield* Pin.withoutLocalPin( - lockPath, - Effect.scoped( - Effect.gen(function* () { - const connection = yield* acquireLock(lockPath, "existing"); - yield* takeExclusiveLock(connection); - const info = yield* fs.stat(lockPath); - const mtime = Option.getOrUndefined(info.mtime); - const now = yield* Clock.currentTimeMillis; - if (mtime === undefined || now - mtime.getTime() <= RETIREMENT_AGE_MILLIS) return false; - yield* relocateToStaging(fs, path, crypto, stagingRoot, generationPath); - return true; - }), - ), - ).pipe( - Effect.map(Option.getOrElse(() => false)), - Effect.orElseSucceed(() => false), - ); + const didRetire = yield* Effect.scoped( + Effect.gen(function* () { + const connection = yield* acquireLock(lockPath, "existing"); + yield* takeExclusiveLock(connection); + const info = yield* fs.stat(lockPath); + const mtime = Option.getOrUndefined(info.mtime); + const now = yield* Clock.currentTimeMillis; + if (mtime === undefined || now - mtime.getTime() <= RETIREMENT_AGE_MILLIS) return false; + yield* relocateToStaging(fs, path, crypto, stagingRoot, generationPath); + return true; + }), + ).pipe(Effect.orElseSucceed(() => false)); if (didRetire) retired++; } yield* Effect.annotateCurrentSpan({ "artifact.retired_count": retired }); @@ -870,11 +863,10 @@ const prepareOrResolve = Effect.fn("ArtifactStore.prepareOrResolve")(function* ( onProgress?: (state: "downloading" | "preparing") => void, ): Effect.fn.Return { yield* ensureDirectory(fs, path, resolved.keyRoot, cacheRoot); - yield* reapStaleStaging(fs, path, resolved.stagingRoot); - yield* retireStaleGenerations(fs, path, crypto, cacheRoot); const hit = yield* checkHit(fs, path, cacheRoot, request, resolved); if (Option.isSome(hit)) return hit.value; - return yield* publishGeneration( + yield* reapStaleStaging(fs, path, resolved.stagingRoot); + const published = yield* publishGeneration( fs, path, crypto, @@ -885,6 +877,8 @@ const prepareOrResolve = Effect.fn("ArtifactStore.prepareOrResolve")(function* ( resolved, onProgress, ); + yield* retireStaleGenerations(fs, path, crypto, cacheRoot); + return published; }); export const makeArtifactStore = Effect.fn("ArtifactStore.makeStore")(function* ( diff --git a/packages/stack/src/preparation/artifacts.integration.test.ts b/packages/stack/src/preparation/artifacts.integration.test.ts index a4473ff2fb..1724c4ff29 100644 --- a/packages/stack/src/preparation/artifacts.integration.test.ts +++ b/packages/stack/src/preparation/artifacts.integration.test.ts @@ -69,8 +69,7 @@ const sourceWriting = (bytes: Uint8Array = archive): ArtifactSource => ({ }).pipe(mapMaterializeError), }); -/** A second, sibling digest under the same key: used to force a retirement sweep as a side effect - * of an unrelated `prepare`, without redownloading or recreating the generation under test. */ +/** A second digest, published by `triggerSweep` under a fresh key per call. */ const archiveAlt = new TextEncoder().encode("archive-alt"); const archiveAltSha256 = createHash("sha256").update(archiveAlt).digest("hex"); const sourceWritingAlt = (): ArtifactSource => ({ @@ -91,6 +90,15 @@ const sourceWritingAlt = (): ArtifactSource => ({ }).pipe(mapMaterializeError), }); +/** A cache miss under a fresh key: the only event that triggers a retirement sweep. */ +const triggerSweep = (root: string) => + Effect.gen(function* () { + const crypto = yield* Crypto.Crypto; + const key = `database/postgres-sweep-${yield* crypto.randomUUIDv4}`; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWritingAlt() }); + return yield* store.prepare({ ...request, key }); + }); + /** Sets `path`'s mtime 31 days in the past: a legitimate test-only lever on retirement's age fence. */ const ageLockFile = (fs: FileSystem.FileSystem, path: string) => Effect.gen(function* () { @@ -731,22 +739,18 @@ describe("pins", () => { expect(firstUse.path).toBe(secondUse.path); yield* ageLockFile(fs, firstUse.lockPath); - const altStore = yield* makeArtifactStore({ - cacheRoot: root, - source: sourceWritingAlt(), - }); // Sweeping while both consumers hold the pin: busy against this process's own open // connection, so the stale generation survives untouched. - yield* altStore.prepare({ ...request, key }); + yield* triggerSweep(root); expect(yield* fs.exists(firstUse.path)).toBe(true); yield* Scope.close(firstScope, Exit.void); - yield* altStore.prepare({ ...request, key }); + yield* triggerSweep(root); expect(yield* fs.exists(firstUse.path)).toBe(true); yield* Scope.close(secondScope, Exit.void); - yield* altStore.prepare({ ...request, key }); + yield* triggerSweep(root); expect(yield* fs.exists(firstUse.path)).toBe(false); expect(yield* fs.exists(firstUse.lockPath)).toBe(true); }), @@ -773,16 +777,12 @@ describe("pins", () => { throw new Error(`Unparseable fixture output: ${line}`); yield* ageLockFile(fs, lockPath); - const altStore = yield* makeArtifactStore({ - cacheRoot: root, - source: sourceWritingAlt(), - }); - yield* altStore.prepare({ ...request, key }); + yield* triggerSweep(root); expect(yield* fs.exists(generationPath)).toBe(true); yield* killAndAwaitExit(handle); yield* ageLockFile(fs, lockPath); - yield* altStore.prepare({ ...request, key }); + yield* triggerSweep(root); expect(yield* fs.exists(generationPath)).toBe(false); expect(yield* fs.exists(lockPath)).toBe(true); }), @@ -807,9 +807,8 @@ describe("pins", () => { // and genuinely eligible for retirement, unlike the pinned scenarios covered above. yield* ageLockFile(fs, prepared.lockPath); - // A sibling digest under the same key forces a sweep as a side effect. - const altStore = yield* makeArtifactStore({ cacheRoot: root, source: sourceWritingAlt() }); - yield* altStore.prepare({ ...request, key }); + // A cache miss under another key forces a sweep as a side effect. + yield* triggerSweep(root); expect(yield* fs.exists(prepared.path)).toBe(false); expect(yield* fs.exists(prepared.lockPath)).toBe(true); @@ -823,6 +822,64 @@ describe("pins", () => { ), ); + it.live( + "a sweep retires an aged unpinned generation and spares an aged one this process pins", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-sweep-pinned-", + }); + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const pinned = yield* Effect.scoped( + Effect.gen(function* () { + const pinnedUse = yield* store.use({ + ...request, + key: "database/postgres-aged-pinned", + }); + const unpinned = yield* store.prepare({ + ...request, + key: "database/postgres-aged-unpinned", + }); + yield* ageLockFile(fs, pinnedUse.lockPath); + yield* ageLockFile(fs, unpinned.lockPath); + + yield* triggerSweep(root); + + expect(yield* fs.exists(unpinned.path)).toBe(false); + expect(yield* fs.exists(pinnedUse.path)).toBe(true); + return pinnedUse; + }), + ); + expect(yield* fs.exists(pinned.path)).toBe(true); + }), + ), + ); + + it.live("a cache hit leaves aged unpinned generations alone until the next miss", () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-hit-no-sweep-", + }); + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const warm = yield* store.prepare({ ...request, key: "database/postgres-warm" }); + const stale = yield* store.prepare({ ...request, key: "database/postgres-stale" }); + yield* ageLockFile(fs, stale.lockPath); + + const hit = yield* store.prepare({ ...request, key: "database/postgres-warm" }); + expect(hit.outcome).toBe("cached"); + expect(hit.path).toBe(warm.path); + expect(yield* fs.exists(stale.path)).toBe(true); + + yield* triggerSweep(root); + expect(yield* fs.exists(stale.path)).toBe(false); + }), + ), + ); + it.live( "a fresh pin never fails busy against a same-process retirement sweep triggered through the real store", () => @@ -948,16 +1005,12 @@ describe("pins", () => { ).pipe(Scope.provide(processScope)); yield* ageLockFile(fs, prepared.lockPath); - const altStore = yield* makeArtifactStore({ - cacheRoot: root, - source: sourceWritingAlt(), - }); - yield* altStore.prepare({ ...request, key }); + yield* triggerSweep(root); expect(yield* fs.exists(prepared.path)).toBe(true); yield* Scope.close(processScope, Exit.void); yield* ageLockFile(fs, prepared.lockPath); - yield* altStore.prepare({ ...request, key }); + yield* triggerSweep(root); expect(yield* fs.exists(prepared.path)).toBe(false); }), ), From fc01575929a6e40965e5edd972d0fae352915d71 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 00:34:26 +0200 Subject: [PATCH 059/142] chore(cli): drop functions deploy error tags from the telemetry snapshot The functions deploy rewrite moved off this branch, so its error tags no longer exist here. --- apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt b/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt index 8dde3a4673..fc5ef75610 100644 --- a/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt +++ b/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt @@ -253,16 +253,10 @@ FeedbackInvalidTokenError FeedbackMessageTooLongError FeedbackNotFoundError FileWatcherError -FunctionAssetIsDirectoryError -FunctionAssetOutsideRootError -FunctionBundleFailedError FunctionDeployCancelledError -FunctionDeployJobsRequiresApiError FunctionDownloadNotFoundError -FunctionImportMapParseError FunctionImportNotDirectoryError FunctionNotFoundError -FunctionStaticPatternNoMatchError FunctionsApiStatusError FunctionsApiTransportError FunctionsEnvNotSupportedError From c251ccf3efd461f7f052179dc0f5d7c35d7ea66b Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 00:42:30 +0200 Subject: [PATCH 060/142] fix(stack): reap orphan staging in every key root and retire generations outside the digest lock A miss now sweeps stale unlocked staging under every key root, so a crashed preparer's leftovers no longer persist behind a published key. Retirement holds the digest lock only for the age re-check and rename into the staging slot; the recursive delete runs under the slot lock alone. Also keeps the port reservation layer alive across the e2e test body and makes the pin test's child report busy only for SQLite errcode 5. --- .../src/namespace/Pin.integration.test.ts | 4 + .../stack/src/port-reservation.e2e.test.ts | 7 +- .../stack/src/preparation/ArtifactStore.ts | 163 +++++++++++------- .../preparation/artifacts.integration.test.ts | 31 +++- 4 files changed, 136 insertions(+), 69 deletions(-) diff --git a/packages/stack/src/namespace/Pin.integration.test.ts b/packages/stack/src/namespace/Pin.integration.test.ts index 683c42cd13..116f4e0c6d 100644 --- a/packages/stack/src/namespace/Pin.integration.test.ts +++ b/packages/stack/src/namespace/Pin.integration.test.ts @@ -17,6 +17,10 @@ const exclusiveLockOutcome = (lockPath: string) => database.exec("BEGIN EXCLUSIVE"); console.log("acquired"); } catch (error) { + if (error.errcode !== 5) { + console.error(error); + process.exit(1); + } console.log("busy"); } `; diff --git a/packages/stack/src/port-reservation.e2e.test.ts b/packages/stack/src/port-reservation.e2e.test.ts index 3e7d87dacd..a4e7452951 100644 --- a/packages/stack/src/port-reservation.e2e.test.ts +++ b/packages/stack/src/port-reservation.e2e.test.ts @@ -1,6 +1,6 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, FileSystem, Layer, Option } from "effect"; +import { Cause, Context, Effect, Exit, FileSystem, Layer, Option } from "effect"; import * as Net from "node:net"; import { create, open, type Observation } from "./effect.ts"; import { ownerExitProbe, waitForOwnerExit } from "./HostProcess.ts"; @@ -391,9 +391,8 @@ it.live( // registry file replaced out from under a stopped stack): the saved claim still names the // port, but no row backs it. const realStateRoot = yield* fs.realPath(stateRootA); - const portReservations = yield* PortReservations.Service.pipe( - Effect.provide(PortReservations.layer), - ); + const reservationContext = yield* Layer.build(PortReservations.layer); + const portReservations = Context.get(reservationContext, PortReservations.Service); yield* portReservations.release(realStateRoot, stackA.id, `${mailA.id}:http`); const reopenedA = yield* open({ id: stackA.id, stateRoot: stateRootA, cacheRoot }); diff --git a/packages/stack/src/preparation/ArtifactStore.ts b/packages/stack/src/preparation/ArtifactStore.ts index 61cdc70ee8..72b2bafdb8 100644 --- a/packages/stack/src/preparation/ArtifactStore.ts +++ b/packages/stack/src/preparation/ArtifactStore.ts @@ -1,4 +1,14 @@ -import { Clock, Crypto, Effect, FileSystem, Option, Path, PlatformError, Predicate } from "effect"; +import { + Clock, + Crypto, + Effect, + FileSystem, + Option, + Path, + PlatformError, + Predicate, + Scope, +} from "effect"; import { HttpClient } from "effect/unstable/http"; import { ChildProcessSpawner } from "effect/unstable/process"; import { ArtifactIntegrityError, PreparationError } from "./Errors.ts"; @@ -575,91 +585,117 @@ const reapStaleStaging = ( ), ); -/** Relocates a to-be-retired generation into a fresh, lock-guarded staging slot, then deletes it. */ -const relocateToStaging = Effect.fn("ArtifactStore.relocateToStaging")(function* ( - fs: FileSystem.FileSystem, - path: Path.Path, - crypto: Crypto.Crypto, - stagingRoot: string, - generationPath: string, -) { - yield* fs.makeDirectory(stagingRoot, { recursive: true, mode: 0o700 }); - const token = yield* crypto.randomUUIDv4; - const stagingLockPath = stagingLockFor(path, stagingRoot, token); - const stagingDir = stagingDirFor(path, stagingRoot, token); - yield* Effect.scoped( - Effect.gen(function* () { - const connection = yield* acquireLock(stagingLockPath, "create"); - yield* takeLock(connection); - yield* fs.rename(generationPath, stagingDir); - yield* fs.remove(stagingDir, { recursive: true, force: true }); - }), - ); - yield* fs - .remove(stagingLockPath, { force: true }) - .pipe(Effect.andThen(fs.remove(`${stagingLockPath}-journal`, { force: true })), Effect.ignore); -}); +interface CacheWalk { + readonly generations: ReadonlyArray<{ readonly keyRoot: string; readonly digest: string }>; + readonly stagingRoots: ReadonlyArray; +} /** - * Recursively discovers every generation (a digest-named directory) anywhere under `root`, - * without ever descending into one: a key embeds its release version (see `Artifacts.ts`'s - * `artifactKey`), so a CLI upgrade leaves a previous release's generations behind a now-unreachable - * key that only a root-wide walk like this ever revisits. A plain directory-name listing, never a - * generation's own (potentially large) published contents, keeps this cheap. + * Recursively walks the cache root and returns every generation (a digest-named directory) and + * every `.staging` root, without ever descending into either: a key embeds its release version + * (see `Artifacts.ts`'s `artifactKey`), so a CLI upgrade leaves a previous release's generations + * behind a now-unreachable key that only a root-wide walk like this ever revisits. A plain + * directory-name listing, never a generation's own (potentially large) published contents, keeps + * this cheap. */ -const discoverGenerations = ( +const walkCache = ( fs: FileSystem.FileSystem, path: Path.Path, root: string, -): Effect.Effect, never> => +): Effect.Effect => Effect.gen(function* () { const names = yield* fs.readDirectory(root).pipe(Effect.orElseSucceed(() => [])); - const found: Array<{ readonly keyRoot: string; readonly digest: string }> = []; + const generations: Array = []; + const stagingRoots: Array = []; for (const name of names) { - if (name === STAGING_DIR_NAME) continue; + if (name === STAGING_DIR_NAME) { + stagingRoots.push(path.join(root, name)); + continue; + } if (DIGEST_NAME.test(name)) { - found.push({ keyRoot: root, digest: name }); + generations.push({ keyRoot: root, digest: name }); continue; } - found.push(...(yield* discoverGenerations(fs, path, path.join(root, name)))); + const nested = yield* walkCache(fs, path, path.join(root, name)); + generations.push(...nested.generations); + stagingRoots.push(...nested.stagingRoots); } - return found; + return { generations, stagingRoots }; }); /** - * One best-effort sweep over every generation published anywhere under the cache root: a - * generation is retired only once an uncontended EXCLUSIVE lock on its digest lock file is taken - * AND, rechecked while holding that lock, its mtime is older than the retention window. Digest - * lock files are never deleted (a stable inode); only the generation directory they guard is. A - * per-digest failure (including contention from a live pin or another retirer) is skipped, never - * propagated: this sweep must never block the `prepare`/`use` call it runs alongside. A pin held - * by this process contends with the sweep's lock exactly as a pin held by another process does. + * Retires one generation. The digest lock is held only for the age re-check and the rename into + * a lock-guarded staging slot, so a concurrent pin is never blocked behind the recursive delete; + * the slot lock keeps the delete from being reaped while it runs. A crash after the rename leaves + * an unlocked staging directory that the next sweep reaps. */ -const retireStaleGenerations = Effect.fn("ArtifactStore.retire")(function* ( +const retireGeneration = Effect.fn("ArtifactStore.retireGeneration")(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + crypto: Crypto.Crypto, + keyRoot: string, + digest: string, +): Effect.fn.Return { + const generationPath = path.join(keyRoot, digest); + const lockPath = path.join(keyRoot, `${digest}.lock`); + const stagingRoot = path.join(keyRoot, STAGING_DIR_NAME); + const stagingLockPath = yield* Effect.scoped( + Effect.gen(function* () { + const slotScope = yield* Scope.Scope; + const slot = yield* Effect.scoped( + Effect.gen(function* () { + const connection = yield* acquireLock(lockPath, "existing"); + yield* takeExclusiveLock(connection); + const info = yield* fs.stat(lockPath); + const mtime = Option.getOrUndefined(info.mtime); + const now = yield* Clock.currentTimeMillis; + if (mtime === undefined || now - mtime.getTime() <= RETIREMENT_AGE_MILLIS) + return undefined; + yield* fs.makeDirectory(stagingRoot, { recursive: true, mode: 0o700 }); + const token = yield* crypto.randomUUIDv4; + const slotLockPath = stagingLockFor(path, stagingRoot, token); + const slotDir = stagingDirFor(path, stagingRoot, token); + const slotConnection = yield* acquireLock(slotLockPath, "create").pipe( + Scope.provide(slotScope), + ); + yield* takeLock(slotConnection); + yield* fs.rename(generationPath, slotDir); + return { slotDir, slotLockPath }; + }), + ); + if (slot === undefined) return undefined; + yield* fs.remove(slot.slotDir, { recursive: true, force: true }); + return slot.slotLockPath; + }), + ).pipe(Effect.orElseSucceed(() => undefined)); + if (stagingLockPath === undefined) return false; + yield* fs + .remove(stagingLockPath, { force: true }) + .pipe(Effect.andThen(fs.remove(`${stagingLockPath}-journal`, { force: true })), Effect.ignore); + return true; +}); + +/** + * One best-effort sweep over the whole cache root. It reaps every unlocked `.staging` entry under + * any key root (a crashed preparer's leftovers), then retires each published generation whose + * digest lock is uncontended and whose lock-file mtime, rechecked under that lock, is older than + * the retention window. Digest lock files are never deleted (a stable inode); only the generation + * directory they guard is. A per-entry failure (including contention from a live pin or another + * retirer) is skipped, never propagated: this sweep must never block the `prepare`/`use` call it + * runs alongside. A pin held by this process contends with the sweep's lock exactly as a pin held + * by another process does. + */ +const sweepCache = Effect.fn("ArtifactStore.sweep")(function* ( fs: FileSystem.FileSystem, path: Path.Path, crypto: Crypto.Crypto, cacheRoot: string, ): Effect.fn.Return { - const generations = yield* discoverGenerations(fs, path, cacheRoot); + const { generations, stagingRoots } = yield* walkCache(fs, path, cacheRoot); + for (const stagingRoot of stagingRoots) yield* reapStaleStaging(fs, path, stagingRoot); let retired = 0; for (const { keyRoot, digest } of generations) { - const generationPath = path.join(keyRoot, digest); - const lockPath = path.join(keyRoot, `${digest}.lock`); - const stagingRoot = path.join(keyRoot, STAGING_DIR_NAME); - const didRetire = yield* Effect.scoped( - Effect.gen(function* () { - const connection = yield* acquireLock(lockPath, "existing"); - yield* takeExclusiveLock(connection); - const info = yield* fs.stat(lockPath); - const mtime = Option.getOrUndefined(info.mtime); - const now = yield* Clock.currentTimeMillis; - if (mtime === undefined || now - mtime.getTime() <= RETIREMENT_AGE_MILLIS) return false; - yield* relocateToStaging(fs, path, crypto, stagingRoot, generationPath); - return true; - }), - ).pipe(Effect.orElseSucceed(() => false)); - if (didRetire) retired++; + if (yield* retireGeneration(fs, path, crypto, keyRoot, digest)) retired++; } yield* Effect.annotateCurrentSpan({ "artifact.retired_count": retired }); return retired; @@ -865,7 +901,6 @@ const prepareOrResolve = Effect.fn("ArtifactStore.prepareOrResolve")(function* ( yield* ensureDirectory(fs, path, resolved.keyRoot, cacheRoot); const hit = yield* checkHit(fs, path, cacheRoot, request, resolved); if (Option.isSome(hit)) return hit.value; - yield* reapStaleStaging(fs, path, resolved.stagingRoot); const published = yield* publishGeneration( fs, path, @@ -877,7 +912,7 @@ const prepareOrResolve = Effect.fn("ArtifactStore.prepareOrResolve")(function* ( resolved, onProgress, ); - yield* retireStaleGenerations(fs, path, crypto, cacheRoot); + yield* sweepCache(fs, path, crypto, cacheRoot); return published; }); diff --git a/packages/stack/src/preparation/artifacts.integration.test.ts b/packages/stack/src/preparation/artifacts.integration.test.ts index 1724c4ff29..7dd581b7c9 100644 --- a/packages/stack/src/preparation/artifacts.integration.test.ts +++ b/packages/stack/src/preparation/artifacts.integration.test.ts @@ -684,6 +684,35 @@ describe("lock-guarded staging", () => { 20_000, ); + it.live( + "reaps a crashed preparer's orphan staging under a published key when a miss runs under another key", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-orphan-reap-", + }); + const key = "database/postgres-orphan-reap"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const published = yield* store.prepare({ ...request, key }); + const orphan = `${root}/${key}/.staging/crashed-token`; + yield* fs.makeDirectory(`${orphan}/content`, { recursive: true }); + yield* fs.writeFileString(`${orphan}/content/partial`, "partial download"); + + const hit = yield* store.prepare({ ...request, key }); + expect(hit.outcome).toBe("cached"); + expect(yield* fs.exists(orphan)).toBe(true); + + yield* triggerSweep(root); + + expect(yield* fs.exists(orphan)).toBe(false); + expect(yield* fs.exists(published.path)).toBe(true); + }), + ), + 20_000, + ); + it.live( "never touches a live preparer's staging directory while another preparer runs", () => @@ -901,7 +930,7 @@ describe("pins", () => { const keyRoot = path.dirname(prepared.path); // Gates the sweep right after it opens its own connection on the target's lock path - // (inside `acquireLock`/`takeExclusiveLock`, just before `retireStaleGenerations`' + // (inside `acquireLock`/`takeExclusiveLock`, just before `retireGeneration`' // `fs.stat` call), holding it open across a real async gap under full test control. const sweepIsHolding = yield* Deferred.make(); const releaseSweep = yield* Deferred.make(); From 8aed0ac9645f373adb23fff7bbdcea2dc15cbd1f Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 00:57:09 +0200 Subject: [PATCH 061/142] refactor(stack): derive the native postgres socket directory instead of journaling claims The native Postgres socket directory is named pg- under the validated per-user runtime root. Launch replaces any leftover at that name and a scoped finalizer removes it on stop. Stop, destroy, the host-startup sweep and orphan reclaim recompute the names and remove them under the recovery-validated runtime root; an invalid root is left alone with a warning. Nothing about the directory is persisted, so the claims journal, its namespace service and the directoryClaims option are deleted. --- packages/stack/README.md | 2 +- .../stack/src/Commands.integration.test.ts | 3 - ...er.native-socket-sweep.integration.test.ts | 167 +++++++++++++ packages/stack/src/Owner.ts | 132 +++------- packages/stack/src/Ports.integration.test.ts | 31 ++- ....directory-claim-sweep.integration.test.ts | 234 ------------------ packages/stack/src/StackHost.ts | 2 +- packages/stack/src/StackNamespace.ts | 8 +- packages/stack/src/Sweep.ts | 2 +- packages/stack/src/namespace/Claims.ts | 157 ------------ .../stack/src/namespace/Namespace.e2e.test.ts | 2 +- packages/stack/src/namespace/Paths.ts | 20 -- packages/stack/src/namespace/Registry.ts | 2 - .../src/runtime/Container.integration.test.ts | 2 +- packages/stack/src/runtime/postgres-user.ts | 50 +++- ...nalyticsVectorImgproxy.integration.test.ts | 3 +- .../services/AuthStorage.integration.test.ts | 3 +- .../src/services/Catalog.integration.test.ts | 3 +- packages/stack/src/services/Catalog.ts | 1 - .../src/services/Database.integration.test.ts | 56 ++++- packages/stack/src/services/Database.ts | 41 ++- .../services/Functions.integration.test.ts | 9 +- .../src/services/Mail.integration.test.ts | 3 +- .../src/services/Pooler.integration.test.ts | 3 +- .../ProcessRecipe.integration.test.ts | 9 +- .../src/services/Realtime.integration.test.ts | 3 +- packages/stack/src/services/Recipe.ts | 3 - .../src/services/Rest.integration.test.ts | 3 +- .../src/services/Studio.integration.test.ts | 3 +- .../src/services/Vector.integration.test.ts | 3 +- packages/stack/tests/claims.ts | 17 -- .../tests/docker-fixture.integration.test.ts | 2 - packages/stack/tests/port-reservations.ts | 7 + 33 files changed, 361 insertions(+), 625 deletions(-) create mode 100644 packages/stack/src/Owner.native-socket-sweep.integration.test.ts delete mode 100644 packages/stack/src/StackHost.directory-claim-sweep.integration.test.ts delete mode 100644 packages/stack/src/namespace/Claims.ts delete mode 100644 packages/stack/tests/claims.ts create mode 100644 packages/stack/tests/port-reservations.ts diff --git a/packages/stack/README.md b/packages/stack/README.md index 3248d8334a..12fb93d4ba 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -89,7 +89,7 @@ The native artifact cache holds one immutable, content-addressed generation per Native PostgreSQL refuses to run as uid 0. When the stack runs as root inside a detected agent sandbox (Claude Code, Modal Sandbox), or `SUPABASE_NATIVE_POSTGRES_USER=` names a non-root system user, only the PostgreSQL process runs as that user: the instance data directory, root key file, socket directory, and the bundle's `pgsodium_getkey.sh` are chowned to it, and the instance directory and its ancestors outside the generation (the artifact cache and stack state roots, which otherwise stay restricted to their owner) receive traverse-only (`o+x`) permission. A directory inside the generation itself is never chmodded; it must already carry that bit from the archive, or the handover fails naming the path. The `pgsodium_getkey.sh` chown is the one accepted exception to generation immutability: the packaged init script hard-codes that cached path and `chmod +x`s it, which only its owner may do. Running as root elsewhere fails before PostgreSQL launches. -Native PostgreSQL listens only on its socket and reads a stack-generated HBA file, written to the socket directory on every launch, instead of `PGDATA/pg_hba.conf`. It trusts `supabase_admin`, including through the proxied loopback database port, and requires `scram-sha-256` passwords from every other role. +Native PostgreSQL listens only on its socket and reads a stack-generated HBA file, written to the socket directory on every launch, instead of `PGDATA/pg_hba.conf`. The socket directory is `pg-` under the validated per-user runtime root, where the digest derives from the data root and instance id, so nothing about it is persisted: launch replaces a leftover, and stop, destroy and orphan recovery remove it. It trusts `supabase_admin`, including through the proxied loopback database port, and requires `scram-sha-256` passwords from every other role. ### Public ports diff --git a/packages/stack/src/Commands.integration.test.ts b/packages/stack/src/Commands.integration.test.ts index e4ac5f7da2..f6f61eb37a 100644 --- a/packages/stack/src/Commands.integration.test.ts +++ b/packages/stack/src/Commands.integration.test.ts @@ -25,7 +25,6 @@ import { makeDatabase } from "./services/Database.ts"; import { makeStandaloneService } from "../tests/standalone-service.ts"; import { bindTcp, serveTcp } from "./Proxy.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; -import { noDirectoryClaims } from "../tests/claims.ts"; import { dockerEngineTarget } from "../tests/engine-target.ts"; const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; @@ -89,7 +88,6 @@ describe("finite PostgreSQL commands", { timeout: 180_000 }, () => { ? yield* makeDockerDatabaseRoot("stack-tools-", stackId) : yield* fs.makeTempDirectoryScoped({ prefix: "stack-tools-" }); const database = yield* makeDatabase({ - directoryClaims: noDirectoryClaims, root, cacheRoot, stackId, @@ -241,7 +239,6 @@ describe("finite PostgreSQL commands", { timeout: 180_000 }, () => { ? yield* makeDockerDatabaseRoot(`stack-pgprove-${major}-`, stackId) : yield* fs.makeTempDirectoryScoped({ prefix: `stack-pgprove-${major}-` }); const database = yield* makeDatabase({ - directoryClaims: noDirectoryClaims, root, cacheRoot, stackId, diff --git a/packages/stack/src/Owner.native-socket-sweep.integration.test.ts b/packages/stack/src/Owner.native-socket-sweep.integration.test.ts new file mode 100644 index 0000000000..95204933f6 --- /dev/null +++ b/packages/stack/src/Owner.native-socket-sweep.integration.test.ts @@ -0,0 +1,167 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { createHash } from "node:crypto"; +import { Context, Effect, FileSystem, Layer, Path, Redacted, Schema } from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import * as Owner from "./Owner.ts"; +import { NativeRuntimeRootBase, nativeRuntimeRootPath } from "./runtime/postgres-user.ts"; +import { ServiceCreation } from "./services/Catalog.ts"; +import * as StackNamespace from "./StackNamespace.ts"; +import { reclaimStack } from "./Sweep.ts"; + +const databaseCreation = Schema.decodeEffect(ServiceCreation)({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("sweep-password"), + jwtSecret: Redacted.make("sweep-integration-jwt-secret-long-enough"), + jwtExpiry: 3600, + }, + endpoints: {}, +}); + +/** A fresh state namespace, a native runtime root base, and a stack with one saved database. */ +const arrange = Effect.fn("arrange")(function* (id: string) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs + .makeTempDirectoryScoped({ prefix: "stack-socket-sweep-" }) + .pipe(Effect.flatMap(fs.realPath)); + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root: `${root}/state` })), + StackNamespace.Service, + ); + const saved: StackNamespace.SavedStack = { + id, + runtime: "native", + identity: { projectRoot: root, branchContext: "main", stackName: id }, + instances: [{ id: "database", creation: yield* databaseCreation }], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + const runtimeRootBase = yield* fs.makeTempDirectoryScoped({ prefix: "stack-socket-sweep-base-" }); + const runtimeRoot = nativeRuntimeRootPath(path, runtimeRootBase, process.getuid?.() ?? 0); + const dataRoot = `${root}/state/${id}/data`; + return { root, state, saved, runtimeRootBase, runtimeRoot, dataRoot }; +}); + +type Arrangement = Effect.Success>; + +/** The socket directory a native database instance gets, derived independently of production code. */ +const socketDirectory = (arrangement: Arrangement, instanceId: string) => + `${arrangement.runtimeRoot}/pg-${createHash("sha256") + .update(`${arrangement.dataRoot}\0${instanceId}`) + .digest("hex") + .slice(0, 16)}`; + +/** A directory left by an owner that was SIGKILLed before its scope could remove it. */ +const leaveSocketDirectory = Effect.fn("leaveSocketDirectory")(function* ( + arrangement: Arrangement, + instanceId: string, +) { + const fs = yield* FileSystem.FileSystem; + const directory = socketDirectory(arrangement, instanceId); + yield* fs.makeDirectory(directory, { recursive: true, mode: 0o700 }); + yield* fs.writeFileString(`${directory}/pg_hba.conf`, "local all all trust\n"); + return directory; +}); + +const ownerFor = (arrangement: Arrangement) => + Layer.build( + Owner.layer({ + saved: arrangement.saved, + root: arrangement.dataRoot, + cacheRoot: `${arrangement.root}/cache`, + }).pipe( + Layer.provide( + Layer.merge( + Layer.succeed(StackNamespace.Service, arrangement.state), + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), + ), + ), + ), + Layer.provide(Layer.succeed(NativeRuntimeRootBase, arrangement.runtimeRootBase)), + ), + ).pipe(Effect.map((context) => Context.get(context, Owner.Service))); + +const platform = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); + +for (const operation of ["stop", "destroy"] as const) + it.live(`${operation} removes the socket directory a killed owner left behind`, () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const arrangement = yield* arrange(`socket-${operation}`); + const leftover = yield* leaveSocketDirectory(arrangement, "database"); + const owner = yield* ownerFor(arrangement); + + yield* owner.namespace[operation]; + + expect(yield* fs.exists(leftover)).toBe(false); + }), + ).pipe(Effect.provide(platform)), + ); + +it.live("an orphan reclaim removes the socket directory of a stack whose owner is gone", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const arrangement = yield* arrange("socket-orphan"); + const leftover = yield* leaveSocketDirectory(arrangement, "database"); + + const reclaimed = yield* reclaimStack({ + state: arrangement.state, + stateRoot: `${arrangement.root}/state`, + cacheRoot: `${arrangement.root}/cache`, + id: arrangement.saved.id, + }).pipe(Effect.provide(Layer.succeed(NativeRuntimeRootBase, arrangement.runtimeRootBase))); + + expect(reclaimed).toBe(true); + expect(yield* fs.exists(leftover)).toBe(false); + }), + ).pipe(Effect.provide(platform)), +); + +it.live("removes only the socket directory of the stack being cleaned up", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cleaned = yield* arrange("socket-cleaned"); + const sibling = yield* arrange("socket-sibling"); + const cleanedLeftover = yield* leaveSocketDirectory(cleaned, "database"); + // Same instance id and runtime root, different stack: a distinct derived name. + const siblingLeftover = yield* leaveSocketDirectory( + { ...sibling, runtimeRoot: cleaned.runtimeRoot }, + "database", + ); + const owner = yield* ownerFor(cleaned); + + yield* owner.namespace.destroy; + + expect(yield* fs.exists(cleanedLeftover)).toBe(false); + expect(yield* fs.exists(siblingLeftover)).toBe(true); + }), + ).pipe(Effect.provide(platform)), +); + +it.live("leaves a socket directory under a runtime root another uid could have taken over", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const arrangement = yield* arrange("socket-takeover"); + const leftover = yield* leaveSocketDirectory(arrangement, "database"); + // Tests cannot create another uid's files, so a world-writable root stands in for one + // recreated by a foreign uid. + yield* fs.chmod(arrangement.runtimeRoot, 0o777); + const owner = yield* ownerFor(arrangement); + + yield* owner.namespace.destroy; + + expect(yield* fs.exists(leftover)).toBe(true); + }), + ).pipe(Effect.provide(platform)), +); diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index fdb059e74d..68d9a9a23b 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -18,7 +18,6 @@ import { } from "effect"; import { HttpClient } from "effect/unstable/http"; import { ChildProcessSpawner } from "effect/unstable/process"; -import type { PlatformError } from "effect/PlatformError"; import type { Rpc, RpcGroup } from "effect/unstable/rpc"; import { failureMessage } from "./internal/failure-message.ts"; import * as Network from "./Network.ts"; @@ -66,11 +65,8 @@ import { } from "./services/Catalog.ts"; import type { CatalogError } from "./services/Recipe.ts"; import * as Container from "./runtime/Container.ts"; -import * as Claims from "./namespace/Claims.ts"; -import { namespaceError } from "./namespace/Capabilities.ts"; -import { lstatPath } from "./namespace/drivers/FileSystem.ts"; import * as Paths from "./namespace/Paths.ts"; -import { resolveNativeRuntimeRootForRecovery } from "./runtime/postgres-user.ts"; +import { removeNativeSocketDirectories } from "./runtime/postgres-user.ts"; import { projectSegmentFor } from "./identity/Identity.ts"; import { stackError, type OwnerRpc } from "./Rpc.ts"; import * as StackNamespace from "./StackNamespace.ts"; @@ -102,56 +98,24 @@ type Handlers = { ) => Rpc.ResultFrom; }; -/** True when `target` exists in any form (a real entry or a symlink); lstat-style, never follows. */ -const directoryClaimExists = Effect.fn("Owner.directoryClaimExists")(function* (target: string) { - return (yield* lstatPath(target)) !== undefined; -}); - /** - * Reconciles this stack's directory claims by exact identity, then removes every container - * carrying this stack's identity and data-root labels through the owner's pinned engine target. - * A directory claim outside every owned root is kept and reported, so recovery (possibly as root) - * never deletes what it does not own; one naming a path that no longer exists in any form is - * simply dropped, nothing left to remove. A container that survives removal fails the sweep, so - * the caller (startup, stop, or destroy) can report it and retry instead of proceeding as if the - * stack were fully torn down. + * Removes the native socket directory of every database instance of this stack, then every + * container carrying this stack's identity and data-root labels through the owner's pinned + * engine target. A container that survives removal fails the sweep, so the caller (startup, + * stop, or destroy) can report it and retry instead of proceeding as if the stack were fully + * torn down. */ export const sweepContainers = Effect.fn("Owner.sweepContainers")(function* ( - state: StackNamespace.Interface, - saved: Pick, + saved: Pick, root: string, target: Container.EngineTarget | undefined, ) { - const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - // Missing, foreign, or symlinked is "no such owned root", not a sweep failure: anything claimed - // under it is reported, not deleted through whatever replaced it. - const nativeRuntimeRoot = yield* resolveNativeRuntimeRootForRecovery().pipe(Effect.option); - const ownedRoots = [root, ...Option.toArray(nativeRuntimeRoot)]; - yield* Claims.reconcile( - state, - saved.id, - (claim): Effect.Effect<"removed" | "kept", PlatformError | StackNamespace.NamespaceError> => - directoryClaimExists(claim.id).pipe( - Effect.flatMap((present) => - !present - ? Effect.succeed("removed" as const) - : Paths.isWithinOwnedRoots(fs, path, claim.id, ownedRoots, (_operation, cause) => - namespaceError("cleanup", cause), - ).pipe( - Effect.flatMap((within) => - within - ? fs - .remove(claim.id, { recursive: true, force: true }) - .pipe(Effect.as("removed" as const)) - : Effect.logWarning( - `Claimed directory ${claim.id} resolves outside every owned root; keeping it`, - ).pipe(Effect.as("kept" as const)), - ), - ), - ), - ), - ); + if (saved.runtime === "native") + yield* removeNativeSocketDirectories( + root, + saved.instances.filter(({ creation }) => creation.service === "database").map(({ id }) => id), + ); if (target === undefined) return; const remaining = yield* Container.removeStackContainers({ target, @@ -183,7 +147,7 @@ export interface Interface { readonly stop: Effect.Effect; /** * Destroys every instance once in-flight definition changes settle, then releases owned - * containers, claims and saved state. + * containers, native socket directories and saved state. */ readonly destroy: Effect.Effect; /** @@ -470,7 +434,6 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { return yield* withCredentials(creation, credentials); }); - const claims = Claims.forStack(options.state, stackId); const recipeFor = (creation: ServiceCreation, id: string) => makeServiceRecipe(creation, { stackId, @@ -480,7 +443,6 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { cacheRoot: options.cacheRoot, runtime, helpers, - directoryClaims: claims.directories, isPubliclyReserved, ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), ...(options.engineTarget === undefined ? {} : { engineTarget: options.engineTarget }), @@ -828,12 +790,15 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ), }; - const sweep = sweepContainers( - options.state, - options.saved, - options.root, - options.engineTarget, - ).pipe(Effect.provideContext(services)); + // The registration's current instances, not the owner's startup snapshot, name the socket + // directories to remove; destroy reads them before it unregisters each instance. + const readCurrent = options.state + .read(stackId) + .pipe(Effect.map((saved) => saved ?? options.saved)); + const sweep = (saved: SavedStack) => + sweepContainers(saved, options.root, options.engineTarget).pipe( + Effect.provideContext(services), + ); const getStackCredentials = readSaved.pipe( Effect.flatMap(({ credentials }) => credentials === undefined @@ -853,45 +818,28 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { // listener scopes only close afterward, through stop/destroy's ordinary teardown below. stop: network.drain.pipe( Effect.andThen(orchestrator.stopNamespace), - Effect.andThen(sweep), + Effect.andThen(readCurrent.pipe(Effect.flatMap(sweep))), definitionGate.withPermits(1), Effect.withSpan("Owner.stopNamespace"), ), - destroy: network.beginDestroy.pipe( - // Deferred for the whole of this destroy: every instance's own teardown below closes its - // listeners as usual, but leaves its reservation rows for `releaseStack` to drop together, - // only once destroy is confirmed to leave nothing behind. `ensuring` below resumes - // immediate per-service deletion again on every exit, success, failure, or interruption - // alike, so a service destruction, sweep, or claim-read failure can never leave deferral - // stuck on for a stack that is still otherwise live. - Effect.andThen(network.drain), - Effect.andThen(orchestrator.destroyNamespace), - Effect.andThen(sweep), - // A directory claim deliberately kept (for example one resolving outside every owned - // root) must not be lost to a full deregistration; the stack stays registered so a later - // acquisition's reconcile can still finish it. - Effect.andThen(options.state.readClaims(stackId)), - Effect.flatMap((remaining) => - remaining.length === 0 - ? removeContainerEnvRoot.pipe( - Effect.andThen(network.releaseStack), - Effect.andThen(options.state.remove(stackId)), - ) - : Effect.gen(function* () { - const claimsPath = path.join(options.state.root, stackId, Claims.CLAIMS_FILE); - const listed = remaining.map((claim) => ` - ${claim.kind} ${claim.id}`).join("\n"); - return yield* new StackNamespace.NamespaceError({ - operation: "destroy", - message: [ - `${remaining.length} resource claim(s) could not be reconciled; stack stays registered for retry.`, - listed, - `To recover: confirm the listed directories are safe to remove, then run \`supabase stack destroy\` again. ` + - `If they are not, remove the listed claim entries from ${claimsPath} and retry.`, - ].join("\n"), - }); - }), + destroy: readCurrent.pipe( + Effect.flatMap((registered) => + network.beginDestroy.pipe( + // Deferred for the whole of this destroy: every instance's own teardown below closes + // its listeners as usual, but leaves its reservation rows for `releaseStack` to drop + // together, only once destroy is confirmed to leave nothing behind. `ensuring` below + // resumes immediate per-service deletion again on every exit, success, failure, or + // interruption alike, so a service destruction or sweep failure can never leave + // deferral stuck on for a stack that is still otherwise live. + Effect.andThen(network.drain), + Effect.andThen(orchestrator.destroyNamespace), + Effect.andThen(sweep(registered)), + Effect.andThen(removeContainerEnvRoot), + Effect.andThen(network.releaseStack), + Effect.andThen(options.state.remove(stackId)), + Effect.ensuring(network.cancelDestroy), + ), ), - Effect.ensuring(network.cancelDestroy), definitionGate.withPermits(1), Effect.withSpan("Owner.destroyNamespace"), ), diff --git a/packages/stack/src/Ports.integration.test.ts b/packages/stack/src/Ports.integration.test.ts index f18a19c8cd..7f25777bde 100644 --- a/packages/stack/src/Ports.integration.test.ts +++ b/packages/stack/src/Ports.integration.test.ts @@ -22,7 +22,9 @@ import { probeVacant, reserveNativePort, } from "./Ports.ts"; +import { systemError } from "effect/PlatformError"; import { bindTcp } from "./Proxy.ts"; +import { CONTAINER_ENV_DIRNAME } from "./namespace/Paths.ts"; import * as PortReservations from "./namespace/PortReservations.ts"; import * as StackNamespace from "./StackNamespace.ts"; import { ownerFor } from "../tests/owner-rpc.ts"; @@ -239,7 +241,7 @@ it.live( ); it.live( - "a stack-wide destroy defers a dedicated row's release; a sweep failure keeps it uncertain, and the next destroy, once resolved, releases it", + "a stack-wide destroy defers a dedicated row's release; a cleanup failure keeps it uncertain, and the next destroy, once resolved, releases it", () => Effect.scoped( Effect.gen(function* () { @@ -259,14 +261,10 @@ it.live( }; yield* state.save(saved); - // A leftover claim the sweep must reconcile before destroy can confirm nothing remains. - const claimed = path.join(root, "data", "claimed"); - yield* fs.makeDirectory(claimed, { recursive: true }); - yield* state.claim(id, { kind: "directory", id: claimed }); - - // Fails removing that one claimed directory on the first attempt only, standing in for any - // concrete sweep failure (an unreachable engine, a locked file, and so on); resolved for the - // retry once the uncertainty would realistically have cleared. + // Fails removing the stack's scratch directory on the first attempt only, standing in for + // any concrete cleanup failure (an unreachable engine, a locked file, and so on); resolved + // for the retry once the uncertainty would realistically have cleared. + const scratch = path.join(root, "data", CONTAINER_ENV_DIRNAME); let resolved = false; const unreliableFileSystem = Layer.effect( FileSystem.FileSystem, @@ -274,9 +272,18 @@ it.live( FileSystem.FileSystem.of({ ...real, remove: (target, options) => - target === claimed && !resolved - ? Effect.die("Injected sweep failure") - : real.remove(target, options), + Effect.suspend(() => + target === scratch && !resolved + ? Effect.fail( + systemError({ + _tag: "PermissionDenied", + module: "test", + method: "remove", + description: "Injected cleanup failure", + }), + ) + : real.remove(target, options), + ), }), ), ).pipe(Layer.provide(NodeFileSystem.layer)); diff --git a/packages/stack/src/StackHost.directory-claim-sweep.integration.test.ts b/packages/stack/src/StackHost.directory-claim-sweep.integration.test.ts deleted file mode 100644 index c9d69f3752..0000000000 --- a/packages/stack/src/StackHost.directory-claim-sweep.integration.test.ts +++ /dev/null @@ -1,234 +0,0 @@ -import { NodeHttpClient, NodeServices } from "@effect/platform-node"; -import { expect, it } from "@effect/vitest"; -import { Context, Effect, FileSystem, Layer, Path } from "effect"; -import { ChildProcessSpawner } from "effect/unstable/process"; -import * as StackNamespace from "./StackNamespace.ts"; -import * as Owner from "./Owner.ts"; -import { NativeRuntimeRootBase, nativeRuntimeRootPath } from "./runtime/postgres-user.ts"; - -const stateFor = (root: string) => - Layer.build(StackNamespace.layer({ root })).pipe( - Effect.map((context) => Context.get(context, StackNamespace.Service)), - ); - -it.live( - "removes a claimed directory left behind under /tmp and unclaims it, for a native stack with no engine", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-directory-sweep-" }); - const state = yield* stateFor(`${root}/state`); - const saved = { - id: "stack", - runtime: "native" as const, - identity: { projectRoot: root, branchContext: "main", stackName: "directory-sweep" }, - instances: [], - lifetime: "detached" as const, - composition: { members: [], dependencies: [] }, - ports: [], - }; - yield* state.save(saved); - // A now-dead owner claimed this short-socket directory, nested under the native runtime - // root, before being killed; not scoped, since the reconcile under test is itself the one - // that removes it. - const runtimeRootBase = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-directory-sweep-runtime-root-", - }); - const runtimeRoot = nativeRuntimeRootPath(path, runtimeRootBase, process.getuid?.() ?? 0); - const socketDirectory = path.join(runtimeRoot, "pg-leftover"); - yield* fs.makeDirectory(socketDirectory, { recursive: true, mode: 0o700 }); - yield* fs.writeFileString(`${socketDirectory}/pg_hba.conf`, "local all all trust\n"); - yield* state.claim(saved.id, { kind: "directory", id: socketDirectory }); - const ownerLayer = Owner.layer({ - saved, - root: `${root}/data`, - cacheRoot: `${root}/cache`, - }).pipe( - Layer.provide( - Layer.merge( - Layer.succeed(StackNamespace.Service, state), - Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), - ), - ), - ), - Layer.provide(Layer.succeed(NativeRuntimeRootBase, runtimeRootBase)), - ); - const owner = yield* Layer.build(ownerLayer).pipe( - Effect.map((context) => Context.get(context, Owner.Service)), - ); - yield* owner.namespace.destroy; - expect(yield* fs.exists(socketDirectory)).toBe(false); - expect(yield* state.readClaims(saved.id)).toEqual([]); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), -); - -it.live( - "keeps and reports a claimed directory that resolves outside every owned root, for a native stack with no engine", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-directory-sweep-outside-", - }); - const state = yield* stateFor(`${root}/state`); - const saved = { - id: "stack", - runtime: "native" as const, - identity: { - projectRoot: root, - branchContext: "main", - stackName: "directory-sweep-outside", - }, - instances: [], - lifetime: "detached" as const, - composition: { members: [], dependencies: [] }, - ports: [], - }; - yield* state.save(saved); - // A foreign directory under neither the data root nor the native runtime root: never ours - // by location, however it got its name, so recovery must leave it alone. - const foreign = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-directory-sweep-foreign-", - }); - yield* fs.writeFileString(`${foreign}/marker`, "not ours\n"); - yield* state.claim(saved.id, { kind: "directory", id: foreign }); - const ownerLayer = Owner.layer({ - saved, - root: `${root}/data`, - cacheRoot: `${root}/cache`, - }).pipe( - Layer.provide( - Layer.merge( - Layer.succeed(StackNamespace.Service, state), - Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), - ), - ), - ), - ); - const owner = yield* Layer.build(ownerLayer).pipe( - Effect.map((context) => Context.get(context, Owner.Service)), - ); - const failure = yield* owner.namespace.destroy.pipe(Effect.flip); - expect(failure.message).toContain(`directory ${foreign}`); - expect(yield* fs.exists(foreign)).toBe(true); - expect(yield* state.read(saved.id)).toBeDefined(); - expect((yield* state.readClaims(saved.id)).map((claim) => claim.id)).toEqual([foreign]); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), -); - -it.live( - "keeps and reports a claim nested under a runtime root a foreign uid could have taken over", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-directory-sweep-takeover-", - }); - const state = yield* stateFor(`${root}/state`); - const saved = { - id: "stack", - runtime: "native" as const, - identity: { - projectRoot: root, - branchContext: "main", - stackName: "directory-sweep-takeover", - }, - instances: [], - lifetime: "detached" as const, - composition: { members: [], dependencies: [] }, - ports: [], - }; - yield* state.save(saved); - // Tests can't create another uid's files, so a group/world-writable leaf stands in for one - // recreated by a foreign uid after the original was removed. - const runtimeRootBase = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-directory-sweep-takeover-base-", - }); - const runtimeRoot = nativeRuntimeRootPath(path, runtimeRootBase, process.getuid?.() ?? 0); - const claimed = path.join(runtimeRoot, "pg-claimed"); - yield* fs.makeDirectory(claimed, { recursive: true }); - yield* fs.chmod(runtimeRoot, 0o777); - yield* fs.writeFileString(`${claimed}/marker`, "not ours\n"); - yield* state.claim(saved.id, { kind: "directory", id: claimed }); - const ownerLayer = Owner.layer({ - saved, - root: `${root}/data`, - cacheRoot: `${root}/cache`, - }).pipe( - Layer.provide( - Layer.merge( - Layer.succeed(StackNamespace.Service, state), - Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), - ), - ), - ), - Layer.provide(Layer.succeed(NativeRuntimeRootBase, runtimeRootBase)), - ); - const owner = yield* Layer.build(ownerLayer).pipe( - Effect.map((context) => Context.get(context, Owner.Service)), - ); - const failure = yield* owner.namespace.destroy.pipe(Effect.flip); - expect(failure.message).toContain(`directory ${claimed}`); - expect(yield* fs.exists(claimed)).toBe(true); - expect((yield* state.readClaims(saved.id)).map((claim) => claim.id)).toEqual([claimed]); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), -); - -it.live("drops a claimed directory that no longer exists, for a native stack with no engine", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-directory-sweep-gone-" }); - const state = yield* stateFor(`${root}/state`); - const saved = { - id: "stack", - runtime: "native" as const, - identity: { projectRoot: root, branchContext: "main", stackName: "directory-sweep-gone" }, - instances: [], - lifetime: "detached" as const, - composition: { members: [], dependencies: [] }, - ports: [], - }; - yield* state.save(saved); - // Never created on disk: the owner that journaled it crashed before the directory existed. - const gone = path.join(root, "pg-never-existed"); - yield* state.claim(saved.id, { kind: "directory", id: gone }); - const ownerLayer = Owner.layer({ - saved, - root: `${root}/data`, - cacheRoot: `${root}/cache`, - }).pipe( - Layer.provide( - Layer.merge( - Layer.succeed(StackNamespace.Service, state), - Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), - ), - ), - ), - ); - const owner = yield* Layer.build(ownerLayer).pipe( - Effect.map((context) => Context.get(context, Owner.Service)), - ); - yield* owner.namespace.destroy; - expect(yield* state.readClaims(saved.id)).toEqual([]); - expect(yield* state.read(saved.id)).toBeUndefined(); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), -); diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index 7e9ee1bcb6..e91320a5bf 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -563,7 +563,7 @@ export const runStackHost = Effect.fn("StackHost.run")( ), ), ); - yield* Owner.sweepContainers(state, saved, dataRoot, engineTarget).pipe( + yield* Owner.sweepContainers(saved, dataRoot, engineTarget).pipe( Effect.mapError((cause) => hostError( "startup-cleanup", diff --git a/packages/stack/src/StackNamespace.ts b/packages/stack/src/StackNamespace.ts index 1f6e540019..0e515e03f2 100644 --- a/packages/stack/src/StackNamespace.ts +++ b/packages/stack/src/StackNamespace.ts @@ -1,5 +1,4 @@ import { Context, Effect, FileSystem, Layer, Path } from "effect"; -import * as Claims from "./namespace/Claims.ts"; import { namespaceError, type NamespaceError } from "./namespace/Capabilities.ts"; import * as Lease from "./namespace/Lease.ts"; import * as Registry from "./namespace/Registry.ts"; @@ -15,8 +14,8 @@ export { StackLifetime, } from "./namespace/Registry.ts"; -export interface Interface extends Registry.Interface, Lease.Interface, Claims.Interface { - /** The namespace's own root directory, so a caller can name a stack's exact claims file. */ +export interface Interface extends Registry.Interface, Lease.Interface { + /** The namespace's own root directory. */ readonly root: string; } @@ -55,8 +54,7 @@ const make = Effect.fn("Namespace.acquire")(function* ( onLeaseContended: options.onLeaseContended, isRegistered: (id) => registry.read(id).pipe(Effect.map((saved) => saved !== undefined)), }); - const claims = yield* Claims.make({ root, platform: options.platform }); - return { ...registry, ...lease, ...claims, root }; + return { ...registry, ...lease, root }; }); export const layer = (options: Options) => diff --git a/packages/stack/src/Sweep.ts b/packages/stack/src/Sweep.ts index 556b114149..ee71312b8c 100644 --- a/packages/stack/src/Sweep.ts +++ b/packages/stack/src/Sweep.ts @@ -60,7 +60,7 @@ export const reclaimStack = Effect.fn("Sweep.reclaimStack")(function* (options: const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const engineTarget = saved.runtime === "native" ? undefined : yield* resolveEngineTarget(spawner); - yield* Owner.sweepContainers(state, saved, dataRoot, engineTarget); + yield* Owner.sweepContainers(saved, dataRoot, engineTarget); } return true; }), diff --git a/packages/stack/src/namespace/Claims.ts b/packages/stack/src/namespace/Claims.ts deleted file mode 100644 index 30e2daf8b7..0000000000 --- a/packages/stack/src/namespace/Claims.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { Effect, FileSystem, Path, Schema, Semaphore } from "effect"; -import { namespaceError, type NamespaceError } from "./Capabilities.ts"; -import * as Publication from "./Publication.ts"; - -/** - * A directory the stack is about to create outside its own process lifetime, whose creation can - * be interrupted partway. Identity is exact, so reconciliation never guesses by name or label. - */ -export const ResourceClaim = Schema.Struct({ - kind: Schema.Literals(["directory"]), - id: Schema.String, -}); -export interface ResourceClaim extends Schema.Schema.Type {} - -const ClaimsDocument = Schema.Struct({ claims: Schema.Array(ResourceClaim) }); - -/** The claims document file name, relative to a stack's own directory under the namespace root. */ -export const CLAIMS_FILE = "claims.json"; - -export interface Interface { - /** Journals `claim` before the resource it names is created; idempotent on a repeated identity. */ - readonly claim: (stackId: string, claim: ResourceClaim) => Effect.Effect; - /** Drops `claim` once its resource is removed, or once something else now owns its cleanup. */ - readonly unclaim: (stackId: string, claim: ResourceClaim) => Effect.Effect; - /** Every resource claim still recorded for `stackId`. */ - readonly readClaims: ( - stackId: string, - ) => Effect.Effect, NamespaceError>; -} - -export interface Options { - readonly root: string; - readonly platform?: NodeJS.Platform; -} - -const sameClaim = (left: ResourceClaim, right: ResourceClaim) => - left.kind === right.kind && left.id === right.id; - -export const make = ( - options: Options, -): Effect.Effect => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const stackRoot = (id: string) => path.join(options.root, id); - const claimsPath = (id: string) => path.join(stackRoot(id), CLAIMS_FILE); - // Claims mutate only from within the single owner process that holds a stack's lease, so an - // in-process mutex per stack id is enough to serialize its read-modify-write cycle. - const locks = new Map(); - const lockFor = (id: string): Semaphore.Semaphore => { - const existing = locks.get(id); - if (existing !== undefined) return existing; - const created = Semaphore.makeUnsafe(1); - locks.set(id, created); - return created; - }; - - const readDocument = (id: string) => - fs.readFileString(claimsPath(id)).pipe( - Effect.mapError((cause) => namespaceError("read", cause)), - Effect.flatMap((text) => - Schema.decodeEffect(Schema.fromJsonString(ClaimsDocument))(text).pipe( - Effect.mapError((cause) => namespaceError("decode", cause)), - ), - ), - Effect.map((document) => document.claims), - ); - const read = (id: string): Effect.Effect, NamespaceError> => - fs.exists(claimsPath(id)).pipe( - Effect.mapError((cause) => namespaceError("read", cause)), - Effect.flatMap((exists) => (exists ? readDocument(id) : Effect.succeed([]))), - ); - - const write = ( - id: string, - claims: ReadonlyArray, - ): Effect.Effect => - fs.makeDirectory(stackRoot(id), { recursive: true, mode: 0o700 }).pipe( - Effect.mapError((cause) => namespaceError("write", cause)), - Effect.andThen( - Schema.encodeEffect(Schema.fromJsonString(ClaimsDocument))({ claims }).pipe( - Effect.mapError((cause) => namespaceError("encode", cause)), - ), - ), - Effect.flatMap((content) => - Publication.publish(fs, path, { - target: claimsPath(id), - content, - platform: options.platform, - }), - ), - ); - - const update = ( - id: string, - next: (current: ReadonlyArray) => ReadonlyArray, - ) => - lockFor(id).withPermits(1)( - read(id).pipe( - Effect.flatMap((current) => { - const updated = next(current); - // An unclaim (or a no-op) that still finds nothing recorded must not recreate the - // stack's directory: the stack may already be fully destroyed, with this call only a - // finalizer running after that, racing the directory it would otherwise resurrect. - if (current.length === 0 && updated.length === 0) return Effect.void; - return write(id, updated); - }), - ), - ); - - const claim = Effect.fn("Namespace.Claims.claim")((stackId: string, claimed: ResourceClaim) => - update(stackId, (current) => - current.some((entry) => sameClaim(entry, claimed)) ? current : [...current, claimed], - ), - ); - const unclaim = Effect.fn("Namespace.Claims.unclaim")( - (stackId: string, claimed: ResourceClaim) => - update(stackId, (current) => current.filter((entry) => !sameClaim(entry, claimed))), - ); - const readClaims = Effect.fn("Namespace.Claims.readClaims")((stackId: string) => read(stackId)); - - return { claim, unclaim, readClaims }; - }); - -/** A narrow capability for journaling a directory outside the owner's normal data tree. */ -export interface DirectoryClaims { - readonly claim: (directoryPath: string) => Effect.Effect; - readonly unclaim: (directoryPath: string) => Effect.Effect; -} - -/** Scopes the claims journal to one stack, exposing only its directory operations. */ -export const forStack = ( - state: Interface, - stackId: string, -): { readonly directories: DirectoryClaims } => ({ - directories: { - claim: (directoryPath) => state.claim(stackId, { kind: "directory", id: directoryPath }), - unclaim: (directoryPath) => state.unclaim(stackId, { kind: "directory", id: directoryPath }), - }, -}); - -/** - * Reads a stack's recorded claims and, for each one still present, asks `remove` to reconcile it: - * `"removed"` drops the claim, `"kept"` leaves it. A claim whose removal fails stays recorded, - * along with every claim after it, for the next acquisition to retry; it reports the first - * failure. - */ -export const reconcile = Effect.fn("Namespace.Claims.reconcile")(function* ( - claimsInterface: Pick, - stackId: string, - remove: (claim: ResourceClaim) => Effect.Effect<"removed" | "kept", E, R>, -): Effect.fn.Return { - for (const claim of yield* claimsInterface.readClaims(stackId)) { - const outcome = yield* remove(claim); - if (outcome === "removed") yield* claimsInterface.unclaim(stackId, claim); - } -}); diff --git a/packages/stack/src/namespace/Namespace.e2e.test.ts b/packages/stack/src/namespace/Namespace.e2e.test.ts index d25b24b405..77068e4bd5 100644 --- a/packages/stack/src/namespace/Namespace.e2e.test.ts +++ b/packages/stack/src/namespace/Namespace.e2e.test.ts @@ -342,7 +342,7 @@ const docker = Effect.fn("Namespace.e2e.docker")((args: ReadonlyArray) = // Needs a real Docker daemon, which only the Linux CI runner has. it.live.skipIf(process.platform === "win32" || process.platform === "darwin")( - "removes exactly the containers a SIGKILLed owner's claims recorded, with no manual cleanup", + "removes exactly the containers a SIGKILLed owner left behind, with no manual cleanup", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/namespace/Paths.ts b/packages/stack/src/namespace/Paths.ts index a6f505f91c..77f1ee33e4 100644 --- a/packages/stack/src/namespace/Paths.ts +++ b/packages/stack/src/namespace/Paths.ts @@ -66,26 +66,6 @@ export const borrow = Effect.fn("Namespace.Paths.borrow")( }), ); -/** Reports whether `candidate`'s real path is inside, or equal to, the real path of any of `roots`. */ -export const isWithinOwnedRoots = Effect.fn("Namespace.Paths.isWithinOwnedRoots")( - ( - fs: FileSystem.FileSystem, - path: Path.Path, - candidate: string, - roots: ReadonlyArray, - onError: (operation: string, cause: unknown) => E, - ): Effect.Effect => - Effect.gen(function* () { - const realCandidate = yield* resolveReal(fs, path, candidate, onError); - for (const root of roots) { - const realRoot = yield* resolveReal(fs, path, root, onError); - if (realCandidate === realRoot || realCandidate.startsWith(`${realRoot}${path.sep}`)) - return true; - } - return false; - }), -); - /** The container mount target for an instance-scoped directory owned on the host. */ export const containerInstancePath = "/instance"; diff --git a/packages/stack/src/namespace/Registry.ts b/packages/stack/src/namespace/Registry.ts index 365b0c9a71..e33c345037 100644 --- a/packages/stack/src/namespace/Registry.ts +++ b/packages/stack/src/namespace/Registry.ts @@ -2,7 +2,6 @@ import { Effect, Exit, FileSystem, Path, Schedule, Schema, Scope } from "effect" import { CompositionConfig } from "../Orchestrator.ts"; import { ServiceCreation } from "../services/Catalog.ts"; import { namespaceError, retryTransientRead, type NamespaceError } from "./Capabilities.ts"; -import { CLAIMS_FILE } from "./Claims.ts"; import { OWNER_FILE, OWNER_LOG_FILE } from "./Lease.ts"; import * as Publication from "./Publication.ts"; import { acquireLock, isBusy, takeLock } from "./drivers/Sqlite.ts"; @@ -188,7 +187,6 @@ export const make = ( statePath(id), path.join(stackRoot(id), OWNER_FILE), path.join(stackRoot(id), OWNER_LOG_FILE), - path.join(stackRoot(id), CLAIMS_FILE), ]) yield* fs .remove(file, { force: true }) diff --git a/packages/stack/src/runtime/Container.integration.test.ts b/packages/stack/src/runtime/Container.integration.test.ts index d7c242cc90..4a3bb724fb 100644 --- a/packages/stack/src/runtime/Container.integration.test.ts +++ b/packages/stack/src/runtime/Container.integration.test.ts @@ -1662,7 +1662,7 @@ describe("container process adapter", () => { ports: [], }; yield* state.save(saved); - yield* Owner.sweepContainers(state, saved, root, target).pipe( + yield* Owner.sweepContainers(saved, root, target).pipe( Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), ); diff --git a/packages/stack/src/runtime/postgres-user.ts b/packages/stack/src/runtime/postgres-user.ts index bb38c05c10..915406d445 100644 --- a/packages/stack/src/runtime/postgres-user.ts +++ b/packages/stack/src/runtime/postgres-user.ts @@ -1,6 +1,7 @@ -import { Config, Context, Effect, FileSystem, Option, Path } from "effect"; +import { Config, Context, Crypto, Effect, FileSystem, Option, Path } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import type { PlatformError } from "effect/PlatformError"; +import { contentDigestHex } from "../internal/content-digest.ts"; import { lstatPath } from "../namespace/drivers/FileSystem.ts"; import { ServiceError } from "../Service.ts"; @@ -229,10 +230,55 @@ export const acquireNativeRuntimeRoot = Effect.fn("NativePostgresUser.acquireRun ); /** Resolves the native runtime root for recovery's containment check, without creating it. */ -export const resolveNativeRuntimeRootForRecovery = Effect.fn( +const resolveNativeRuntimeRootForRecovery = Effect.fn( "NativePostgresUser.resolveRuntimeRootForRecovery", )(() => resolveNativeRuntimeRoot(false)); +/** + * The socket directory of one native database instance: a function of its data root and id alone, + * so launch and recovery agree on it with nothing persisted. PostgreSQL limits a Unix socket path + * (this directory plus `/.s.PGSQL.5432`) to 103 bytes, which bounds the runtime root base. + */ +export const nativeSocketDirectoryPath = ( + crypto: Crypto.Crypto, + path: Path.Path, + runtimeRoot: string, + dataRoot: string, + instanceId: string, +) => + contentDigestHex(crypto, `${dataRoot}\0${instanceId}`).pipe( + Effect.map((digest) => path.join(runtimeRoot, `pg-${digest}`)), + ); + +/** + * Removes the socket directories of a stack's native database instances. A runtime root that is + * missing is nothing to remove; one that is not safe to trust (a symlink, foreign owner or + * writable by others) is warned about and left untouched. + */ +export const removeNativeSocketDirectories = Effect.fn( + "NativePostgresUser.removeSocketDirectories", +)(function* (dataRoot: string, instanceIds: ReadonlyArray) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const runtimeRoot = yield* resolveNativeRuntimeRootForRecovery().pipe(Effect.option); + if (Option.isNone(runtimeRoot)) { + const base = yield* NativeRuntimeRootBase; + const leaf = nativeRuntimeRootPath(path, base, process.getuid?.() ?? 0); + if ((yield* lstatPath(leaf)) !== undefined) + yield* Effect.logWarning( + `${leaf} is not a trusted native runtime root; leaving its socket directories`, + ); + return; + } + yield* Effect.annotateCurrentSpan({ "instance.count": instanceIds.length }); + for (const instanceId of instanceIds) + yield* fs.remove( + yield* nativeSocketDirectoryPath(crypto, path, runtimeRoot.value, dataRoot, instanceId), + { recursive: true, force: true }, + ); +}); + /** * Restricts a directory to its owner but keeps an existing traverse-only grant, because a * stepped-down PostgreSQL resolves bundle and data paths through the cache and state roots while it runs. diff --git a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts index 6c74eca221..274f5df502 100644 --- a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts +++ b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts @@ -7,7 +7,7 @@ import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; -import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; +import { noPublicPortReservations } from "../../tests/port-reservations.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ @@ -16,7 +16,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }); diff --git a/packages/stack/src/services/AuthStorage.integration.test.ts b/packages/stack/src/services/AuthStorage.integration.test.ts index b06f83a5d2..7d25742d24 100644 --- a/packages/stack/src/services/AuthStorage.integration.test.ts +++ b/packages/stack/src/services/AuthStorage.integration.test.ts @@ -8,7 +8,7 @@ import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; -import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; +import { noPublicPortReservations } from "../../tests/port-reservations.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ @@ -17,7 +17,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }); diff --git a/packages/stack/src/services/Catalog.integration.test.ts b/packages/stack/src/services/Catalog.integration.test.ts index bdf9bf0ce1..0e0165b297 100644 --- a/packages/stack/src/services/Catalog.integration.test.ts +++ b/packages/stack/src/services/Catalog.integration.test.ts @@ -2,7 +2,7 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Effect, Exit, FileSystem, Layer, Redacted } from "effect"; import { makeServiceRecipe } from "./Catalog.ts"; -import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; +import { noPublicPortReservations } from "../../tests/port-reservations.ts"; const options = (root: string) => ({ stackId: "catalog-recipe", @@ -10,7 +10,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }); diff --git a/packages/stack/src/services/Catalog.ts b/packages/stack/src/services/Catalog.ts index bbd735bd9e..7924379422 100644 --- a/packages/stack/src/services/Catalog.ts +++ b/packages/stack/src/services/Catalog.ts @@ -339,7 +339,6 @@ export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( ...(options.helpers === undefined ? {} : { helpers: options.helpers }), ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), ...(options.engineTarget === undefined ? {} : { engineTarget: options.engineTarget }), - directoryClaims: options.directoryClaims, }).pipe( Effect.mapError( (cause) => diff --git a/packages/stack/src/services/Database.integration.test.ts b/packages/stack/src/services/Database.integration.test.ts index 047161e2a4..7e2ba6dd87 100644 --- a/packages/stack/src/services/Database.integration.test.ts +++ b/packages/stack/src/services/Database.integration.test.ts @@ -13,12 +13,12 @@ import { Ref, Stream, } from "effect"; +import { createHash } from "node:crypto"; import { tmpdir } from "node:os"; import { DEFAULT_POSTGRES_ROOT_KEY } from "../Defaults.ts"; import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeDatabase, type BackendEndpoint, type DatabaseConfig } from "./Database.ts"; import { makeDockerDatabaseRoot, runDocker } from "../../tests/docker-fixture.ts"; -import { noDirectoryClaims } from "../../tests/claims.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const config: DatabaseConfig = { @@ -72,7 +72,6 @@ describe("database component", { timeout: 180_000 }, () => { ? yield* makeDockerDatabaseRoot("stack-default-key-", "default-key-test") : yield* fs.makeTempDirectoryScoped({ prefix: "stack-default-key-" }); const recipe = yield* makeDatabase({ - directoryClaims: noDirectoryClaims, stackId: "default-key-test", instanceId: "database", root, @@ -132,7 +131,6 @@ describe("database component", { timeout: 180_000 }, () => { ).toEqual([{ decrypted_secret: "preserved-value" }]); if (target.runtime === "native") { const second = yield* makeDatabase({ - directoryClaims: noDirectoryClaims, stackId: "default-key-test", instanceId: "database-second", root, @@ -167,7 +165,6 @@ describe("database component", { timeout: 180_000 }, () => { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-database-hba-" }); const database = yield* makeDatabase({ - directoryClaims: noDirectoryClaims, stackId: "stack-integration", instanceId: "hba", root, @@ -220,6 +217,50 @@ describe("database component", { timeout: 180_000 }, () => { ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); + it.live( + "replaces a socket directory a killed owner left behind and removes its own on stop", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-database-leftover-" }); + // The derived name is computed here independently of production code. + const socketDirectory = `${yield* fs.realPath("/tmp")}/supabase-${process.getuid?.() ?? 0}/pg-${createHash( + "sha256", + ) + .update(`${root}\0leftover`) + .digest("hex") + .slice(0, 16)}`; + yield* fs.makeDirectory(socketDirectory, { recursive: true, mode: 0o700 }); + yield* fs.writeFileString(`${socketDirectory}/stale.marker`, "left by a killed owner\n"); + const database = yield* makeDatabase({ + stackId: "stack-integration", + instanceId: "leftover", + root, + cacheRoot: artifactCacheRoot, + runtime: "native", + }); + const service = yield* makeStandaloneService(database.definition, { + id: "database:leftover", + config, + }); + + yield* service.start; + yield* service.ready; + + const endpoint = yield* database.endpoint; + expect(endpoint).toMatchObject({ kind: "unix", path: socketDirectory }); + expect(yield* fs.exists(`${socketDirectory}/stale.marker`)).toBe(false); + expect(yield* query(endpoint, config.databasePassword, "SELECT 1 AS ok")).toEqual([ + { ok: 1 }, + ]); + yield* service.stop; + expect(yield* fs.exists(socketDirectory)).toBe(false); + yield* service.destroy; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); + it.live( "persists SQL data across exact-session stop and reopen, isolates instances, and validates restart before stopping", () => @@ -230,7 +271,6 @@ describe("database component", { timeout: 180_000 }, () => { const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-database-" }); const cacheRoot = artifactCacheRoot; const first = yield* makeDatabase({ - directoryClaims: noDirectoryClaims, stackId: "stack-integration", instanceId: "first", root, @@ -307,7 +347,6 @@ describe("database component", { timeout: 180_000 }, () => { expect(incomplete.message).toContain("major does not match"); const reopened = yield* makeDatabase({ - directoryClaims: noDirectoryClaims, stackId: "stack-integration", instanceId: "first", root, @@ -334,7 +373,6 @@ describe("database component", { timeout: 180_000 }, () => { const secondRoot = path.join(root, "second-root"); const second = yield* makeDatabase({ - directoryClaims: noDirectoryClaims, stackId: "stack-integration", instanceId: "second", root: secondRoot, @@ -389,7 +427,6 @@ describe("database component", { timeout: 180_000 }, () => { ); const cacheRoot = artifactCacheRoot; const database = yield* makeDatabase({ - directoryClaims: noDirectoryClaims, stackId: "stack-integration-container", instanceId: "database", root, @@ -446,7 +483,6 @@ describe("database component", { timeout: 180_000 }, () => { expect((yield* service.get).lifecycle).toBe("running"); yield* service.stop; const reopened = yield* makeDatabase({ - directoryClaims: noDirectoryClaims, stackId: "stack-integration-container", instanceId: "database", root, @@ -483,7 +519,6 @@ describe("database component", { timeout: 180_000 }, () => { const stackId = "stack-compose-group"; const root = yield* makeDockerDatabaseRoot("stack-database-group-", stackId); const database = yield* makeDatabase({ - directoryClaims: noDirectoryClaims, stackId, instanceId: "database", project: "my.app", @@ -528,7 +563,6 @@ describe("database component", { timeout: 180_000 }, () => { const stackId = "stack-fast-shutdown"; const root = yield* makeDockerDatabaseRoot("stack-database-shutdown-", stackId); const database = yield* makeDatabase({ - directoryClaims: noDirectoryClaims, stackId, instanceId: "database", root, diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index 3663d29c8d..98652ad338 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -62,13 +62,13 @@ import { import type { StackId } from "../identity/StackId.ts"; import { acquireNativeRuntimeRoot, + nativeSocketDirectoryPath, handOverNativePostgresFiles, openNativePostgresInstance, resolveNativePostgresUser, type PasswdEntry, } from "../runtime/postgres-user.ts"; import { EndpointIntent, serviceCreation } from "./Recipe.ts"; -import type * as Claims from "../namespace/Claims.ts"; import * as Environment from "../namespace/Environment.ts"; import { containerInstancePath, destroyOwnedRoot } from "../namespace/Paths.ts"; import { DEFAULT_POSTGRES_ROOT_KEY } from "../Defaults.ts"; @@ -145,8 +145,6 @@ export interface DatabaseOptions { readonly hostGateway?: HostGateway; /** The engine endpoint and identity the owner resolved once at startup; absent when native. */ readonly engineTarget?: EngineTarget; - /** Journals the native socket directory this database creates under `/tmp`, outside its data root. */ - readonly directoryClaims: Claims.DirectoryClaims; } export interface DatabaseComponent { @@ -798,31 +796,24 @@ export const makeDatabase = ( // PostgreSQL limits Unix socket paths to 103 bytes, so this directory nests under the // short per-uid native runtime root (acquired and validated below) rather than under // instanceRoot. Ownership by location there keeps a foreign uid from ever creating or - // replacing it, and keeps recovery from ever having to delete outside it. - // The pathname is picked and journaled before the directory exists, so a SIGKILL - // between the claim and the create always leaves reconcile a recoverable claim. - // An existing path is never ours, so a failed exclusive create drops the claim. + // replacing it, and its derived name lets recovery find a leftover without a journal. const runtimeRoot = yield* asRoot(acquireNativeRuntimeRoot()); - const socketSuffix = yield* crypto.randomUUIDv4.pipe( - Effect.map((uuid) => uuid.replaceAll("-", "")), - Effect.mapError((cause) => errorFor("launch", cause)), - ); - const socketPath = path.join(runtimeRoot, `pg-${socketSuffix}`); + const socketPath = yield* nativeSocketDirectoryPath( + crypto, + path, + runtimeRoot, + options.root, + options.instanceId, + ).pipe(Effect.mapError((cause) => errorFor("launch", cause))); + // A directory already at the derived name is a previous run's leftover. yield* Effect.acquireRelease( - options.directoryClaims - .claim(socketPath) - .pipe( - Effect.andThen( - fs - .makeDirectory(socketPath, { mode: 0o700 }) - .pipe(Effect.tapError(() => options.directoryClaims.unclaim(socketPath))), - ), - ), + fs + .remove(socketPath, { recursive: true, force: true }) + .pipe(Effect.andThen(fs.makeDirectory(socketPath, { mode: 0o700 }))), () => - fs.remove(socketPath, { recursive: true, force: true }).pipe( - Effect.andThen(options.directoryClaims.unclaim(socketPath)), - Effect.catch((cause) => Effect.logError(cause)), - ), + fs + .remove(socketPath, { recursive: true, force: true }) + .pipe(Effect.catch((cause) => Effect.logError(cause))), ).pipe( Scope.provide(context.scope), Effect.mapError((cause) => errorFor("launch", cause)), diff --git a/packages/stack/src/services/Functions.integration.test.ts b/packages/stack/src/services/Functions.integration.test.ts index 7cfed8c6a7..a9b9ccee6e 100644 --- a/packages/stack/src/services/Functions.integration.test.ts +++ b/packages/stack/src/services/Functions.integration.test.ts @@ -22,7 +22,7 @@ import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import * as Functions from "./Functions.ts"; import { makeProcessRecipe } from "./ProcessRecipe.ts"; -import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; +import { noPublicPortReservations } from "../../tests/port-reservations.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; @@ -103,7 +103,6 @@ describe("service catalog", () => { }, }, { - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, ...dockerOptions(root), stackId, @@ -167,7 +166,6 @@ describe("service catalog", () => { config: { functionsRoot, databaseUrl: "postgres://functions-db" }, }, { - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, ...options(root), stackId, @@ -262,7 +260,6 @@ describe("service catalog", () => { }, }, { - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, ...options(stackRoot), stackId: "d".repeat(64), @@ -337,7 +334,6 @@ describe("service catalog", () => { }, }, { - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, ...options(stackRoot), stackId: "e".repeat(64), @@ -413,7 +409,6 @@ describe("service catalog", () => { }, }, { - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, ...options(stackRoot), stackId: "f".repeat(64), @@ -528,7 +523,6 @@ for (const runtime of ["native", "docker"] as const) { }, }, { - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, ...options(stackRoot), stackId: "c".repeat(64), @@ -651,7 +645,6 @@ it.effect("passes POSIX project paths to a docker Functions container from a Win const recipe = yield* makeProcessRecipe( creation, { - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, stackId: "e".repeat(64), instanceId: "windows", diff --git a/packages/stack/src/services/Mail.integration.test.ts b/packages/stack/src/services/Mail.integration.test.ts index e64820af7b..db6ee37ca1 100644 --- a/packages/stack/src/services/Mail.integration.test.ts +++ b/packages/stack/src/services/Mail.integration.test.ts @@ -4,7 +4,7 @@ import { Effect, Exit, FileSystem, Layer, Path } from "effect"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; -import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; +import { noPublicPortReservations } from "../../tests/port-reservations.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; @@ -14,7 +14,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }); diff --git a/packages/stack/src/services/Pooler.integration.test.ts b/packages/stack/src/services/Pooler.integration.test.ts index e61461fc13..02a621a407 100644 --- a/packages/stack/src/services/Pooler.integration.test.ts +++ b/packages/stack/src/services/Pooler.integration.test.ts @@ -7,7 +7,7 @@ import { ProxyError } from "../Proxy.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; +import { noPublicPortReservations } from "../../tests/port-reservations.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ @@ -16,7 +16,6 @@ const options = (root: string) => ({ root, cacheRoot: "/tmp/supabase-stack-artifacts", runtime: "native" as const, - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }); diff --git a/packages/stack/src/services/ProcessRecipe.integration.test.ts b/packages/stack/src/services/ProcessRecipe.integration.test.ts index 1d4b54bdab..3ada73f3e9 100644 --- a/packages/stack/src/services/ProcessRecipe.integration.test.ts +++ b/packages/stack/src/services/ProcessRecipe.integration.test.ts @@ -49,7 +49,7 @@ import { import type { CatalogOptions, RecipeCreation } from "./Recipe.ts"; import * as Realtime from "./Realtime.ts"; import * as Pooler from "./Pooler.ts"; -import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; +import { noPublicPortReservations } from "../../tests/port-reservations.ts"; type TestCreation = RecipeCreation<"rest", Record> & { readonly service: "rest"; @@ -77,7 +77,6 @@ const options: CatalogOptions = { root: "/tmp/process-recipe-test", cacheRoot: "/tmp/process-recipe-test-cache", runtime: "docker", - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }; @@ -461,7 +460,6 @@ const realtimeService = Effect.fn(function* (container: ContainerRuntime) { const recipe = yield* makeProcessRecipe( creation, { - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-test", instanceId: "instance", @@ -959,7 +957,6 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-port-race", instanceId: "instance", @@ -1039,7 +1036,6 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-port-exhaustion", instanceId: "instance", @@ -1180,7 +1176,6 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-deadline", instanceId: "instance", @@ -1246,7 +1241,6 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-unrelated-failure", instanceId: "instance", @@ -1354,7 +1348,6 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-exit-failure", instanceId: "instance", diff --git a/packages/stack/src/services/Realtime.integration.test.ts b/packages/stack/src/services/Realtime.integration.test.ts index d69ee032ed..03fba42f29 100644 --- a/packages/stack/src/services/Realtime.integration.test.ts +++ b/packages/stack/src/services/Realtime.integration.test.ts @@ -6,7 +6,7 @@ import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; +import { noPublicPortReservations } from "../../tests/port-reservations.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; @@ -16,7 +16,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }); diff --git a/packages/stack/src/services/Recipe.ts b/packages/stack/src/services/Recipe.ts index 38bbe8de45..b3c0f02191 100644 --- a/packages/stack/src/services/Recipe.ts +++ b/packages/stack/src/services/Recipe.ts @@ -3,7 +3,6 @@ import type { Stream } from "effect"; import type { Effect, Ref } from "effect"; import type { ServiceKind } from "../Artifacts.ts"; import type { ServiceDefinition } from "../Service.ts"; -import type { DirectoryClaims } from "../namespace/Claims.ts"; import type { PortError } from "../Ports.ts"; import type { EngineTarget, HostGateway } from "../runtime/Container.ts"; import type { DockerHelperRegistry } from "../storage/DockerHelperRegistry.ts"; @@ -58,8 +57,6 @@ export interface CatalogOptions { readonly hostGateway?: HostGateway; /** The engine endpoint and identity the owner resolved once at startup; absent when native. */ readonly engineTarget?: EngineTarget; - /** Journals directories a service creates outside its data root (for example under `/tmp`). */ - readonly directoryClaims: DirectoryClaims; /** Read-only: true when the per-user registry holds `port` for any stack's public listener. */ readonly isPubliclyReserved: (port: number) => Effect.Effect; } diff --git a/packages/stack/src/services/Rest.integration.test.ts b/packages/stack/src/services/Rest.integration.test.ts index 7658aa96b6..0e4bfc97c3 100644 --- a/packages/stack/src/services/Rest.integration.test.ts +++ b/packages/stack/src/services/Rest.integration.test.ts @@ -11,7 +11,7 @@ import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { ProxyError } from "../Proxy.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; +import { noPublicPortReservations } from "../../tests/port-reservations.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; @@ -37,7 +37,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }); diff --git a/packages/stack/src/services/Studio.integration.test.ts b/packages/stack/src/services/Studio.integration.test.ts index 18f9670382..8a26fdad59 100644 --- a/packages/stack/src/services/Studio.integration.test.ts +++ b/packages/stack/src/services/Studio.integration.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Layer } from "effect"; import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; -import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; +import { noPublicPortReservations } from "../../tests/port-reservations.ts"; const options = (root: string) => ({ stackId: "catalog-studio", @@ -11,7 +11,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }); diff --git a/packages/stack/src/services/Vector.integration.test.ts b/packages/stack/src/services/Vector.integration.test.ts index 1b022030a0..ebf2215ffb 100644 --- a/packages/stack/src/services/Vector.integration.test.ts +++ b/packages/stack/src/services/Vector.integration.test.ts @@ -5,7 +5,7 @@ import { Effect, FileSystem, Layer } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; -import { noDirectoryClaims, noPublicPortReservations } from "../../tests/claims.ts"; +import { noPublicPortReservations } from "../../tests/port-reservations.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; @@ -16,7 +16,6 @@ const options = (root: string, runtime: "docker" | "native") => ({ cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, runtime, ...(runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), - directoryClaims: noDirectoryClaims, isPubliclyReserved: noPublicPortReservations, }); diff --git a/packages/stack/tests/claims.ts b/packages/stack/tests/claims.ts deleted file mode 100644 index 67d2d6ca6f..0000000000 --- a/packages/stack/tests/claims.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Effect } from "effect"; -import type { DirectoryClaims } from "../src/namespace/Claims.ts"; -import type { PortError } from "../src/Ports.ts"; - -/** - * No-op claims for tests that exercise directory creation without exercising the namespace's - * reconcile loop itself (covered separately by the namespace's own test suite). - */ -export const noDirectoryClaims: DirectoryClaims = { - claim: () => Effect.void, - unclaim: () => Effect.void, -}; - -/** Stands in for the real per-user registry check in tests not exercising it (covered separately - * by `Ports.integration.test.ts`). */ -export const noPublicPortReservations = (_port: number): Effect.Effect => - Effect.succeed(false); diff --git a/packages/stack/tests/docker-fixture.integration.test.ts b/packages/stack/tests/docker-fixture.integration.test.ts index 8fccf5a18f..cecbd1d666 100644 --- a/packages/stack/tests/docker-fixture.integration.test.ts +++ b/packages/stack/tests/docker-fixture.integration.test.ts @@ -6,7 +6,6 @@ import { Context, Effect, FileSystem, Layer, Redacted, Schema, Stream } from "ef import { makeStandaloneService } from "./standalone-service.ts"; import { makeDatabase, type BackendEndpoint } from "../src/services/Database.ts"; import { makeDockerDatabaseRoot } from "./docker-fixture.ts"; -import { noDirectoryClaims } from "./claims.ts"; import { dockerEngineTarget } from "./engine-target.ts"; const marker = Schema.Struct({ backend: Schema.Literal("docker"), volume: Schema.String }); @@ -76,7 +75,6 @@ describe("Docker database fixture isolation", { timeout: 180_000 }, () => { cacheRoot: `${root}/cache`, runtime: "docker", engineTarget: dockerEngineTarget, - directoryClaims: noDirectoryClaims, }); const [first, second] = yield* Effect.all([make(firstRoot), make(secondRoot)]); const [firstService, secondService] = yield* Effect.all([ diff --git a/packages/stack/tests/port-reservations.ts b/packages/stack/tests/port-reservations.ts new file mode 100644 index 0000000000..8cc756415a --- /dev/null +++ b/packages/stack/tests/port-reservations.ts @@ -0,0 +1,7 @@ +import { Effect } from "effect"; +import type { PortError } from "../src/Ports.ts"; + +/** Stands in for the real per-user registry check in tests not exercising it (covered separately + * by `Ports.integration.test.ts`). */ +export const noPublicPortReservations = (_port: number): Effect.Effect => + Effect.succeed(false); From 11c121ddf0df0b920e2d3e14dbe5c27b7c08b538 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 00:59:08 +0200 Subject: [PATCH 062/142] test(stack): prove a pin is not blocked by a retirement's slow delete The digest lock is held only for the age re-check and the rename into a staging slot, so a consumer pinning the same digest completes while the recursive delete is still in flight. --- .../preparation/artifacts.integration.test.ts | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/packages/stack/src/preparation/artifacts.integration.test.ts b/packages/stack/src/preparation/artifacts.integration.test.ts index 7dd581b7c9..12651b74bc 100644 --- a/packages/stack/src/preparation/artifacts.integration.test.ts +++ b/packages/stack/src/preparation/artifacts.integration.test.ts @@ -851,6 +851,63 @@ describe("pins", () => { ), ); + it.live("a consumer pins a generation while its retirement is still deleting the old copy", () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-retire-delete-", + }); + const key = "database/postgres-retire-delete"; + const setupStore = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const prepared = yield* setupStore.prepare({ ...request, key }); + yield* ageLockFile(fs, prepared.lockPath); + const keyRoot = path.dirname(prepared.path); + + const deleteStarted = yield* Deferred.make(); + const releaseDelete = yield* Deferred.make(); + const sweepingStore = yield* makeArtifactStore({ + cacheRoot: root, + source: sourceWritingAlt(), + }).pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fs, + remove: (...args: Parameters) => + args[0].startsWith(`${keyRoot}/.staging/`) && !isStagingLockFile(args[0]) + ? Deferred.succeed(deleteStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseDelete)), + Effect.andThen(fs.remove(...args)), + ) + : fs.remove(...args), + }), + ); + yield* Effect.addFinalizer(() => Deferred.succeed(releaseDelete, undefined)); + + const sweepFiber = yield* sweepingStore + .prepare({ ...request, key: "database/postgres-retire-delete-trigger" }) + .pipe(Effect.forkScoped); + yield* Deferred.await(deleteStarted); + expect(yield* digestDirectories(fs, keyRoot)).toEqual([]); + + const consumerStore = yield* makeArtifactStore({ + cacheRoot: root, + source: sourceWriting(), + }); + const used = yield* Effect.scoped(consumerStore.use({ ...request, key })); + expect(used.path).toBe(prepared.path); + expect(yield* fs.exists(`${used.path}/bin/postgres`)).toBe(true); + + yield* Deferred.succeed(releaseDelete, undefined); + yield* Fiber.join(sweepFiber); + const leftovers = yield* fs + .readDirectory(`${keyRoot}/.staging`) + .pipe(Effect.map((names) => names.filter((name) => !isStagingLockFile(name)))); + expect(leftovers).toEqual([]); + }), + ), + ); + it.live( "a sweep retires an aged unpinned generation and spares an aged one this process pins", () => From c111e5d83ab8fc791a1ff07c58c7e4ef3603f327 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 01:11:30 +0200 Subject: [PATCH 063/142] fix(stack): tie native socket directory removal to instance destroy and canonical data roots Removing a native database instance's data now also removes its derived socket directory, before the instance is unregistered, so a failed removal keeps the registration as the retry handle. Destroy of a stack and of a single instance share that path; an untrusted runtime root is warned about and left in place. The stack-level socket sweep on stop, startup and reclaim of detached stacks is dropped: launch replaces a leftover and the scoped finalizer removes it on stop. Host startup and orphan reclaim resolve the stack data root through one helper that creates the data directory when missing and resolves symlinks, so cleanup hashes the same path launch does. Docs describe derived socket-directory cleanup instead of directory claims. --- .../stack/destroy/SIDE_EFFECTS.md | 8 +- packages/stack/README.md | 4 +- ...er.native-socket-sweep.integration.test.ts | 190 ++++++++++++++---- packages/stack/src/Owner.ts | 63 +++--- packages/stack/src/StackHost.ts | 4 +- packages/stack/src/StackNamespace.ts | 1 + packages/stack/src/Sweep.ts | 8 +- packages/stack/src/namespace/Paths.ts | 17 ++ packages/stack/src/runtime/postgres-user.ts | 42 ++-- packages/stack/src/services/Database.ts | 27 ++- 10 files changed, 240 insertions(+), 124 deletions(-) diff --git a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md index d48d837d7d..0bd3d3d165 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md @@ -32,10 +32,12 @@ reached, destruction removes nothing: the registration, port claims and host data stay as they are, stderr warns that Docker resources were not removed, and the command exits 0 reporting the stack as not destroyed. Run destroy again once Docker is reachable. Any other engine failure, an owner starting during -destruction, a directory claim the owner cannot reconcile (for example one -outside every owned root), or a labelled container that survives removal, +destruction, an instance whose data (including a native database's socket +directory) cannot be removed, or a labelled container that survives removal, fails the command and keeps the stack registered; the error lists what -remains, including the `claims.json` path for any directory claims. +remains. A native runtime root that is not trusted (a symlink, owned by another +user or writable by others) is warned about and its socket directories are left +in place. ## Files and network diff --git a/packages/stack/README.md b/packages/stack/README.md index 12fb93d4ba..7e733a2de0 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -77,7 +77,7 @@ Pass `startOwner: true` to `open` when live status and other owner-backed operat `create({ ..., startOwner: true })` starts a detached stack's owner immediately and lets it register the stack under its lease, as a session stack always does. If the owner fails to start or the launch is interrupted before it reports ready, the owner removes that registration and `create` fails with the launch error, so a failed launch leaves no stack behind. -`destroy` normally returns `{ runtimeCleanup: "complete" }`. When no owner is running and a new owner cannot start because the stack's container engine reports that its daemon cannot be reached, `destroy` confirms that no owner holds the stack's lease and leaves the registration, port claims and host data untouched, returning `{ runtimeCleanup: "skipped", engine }`. When an owner does run but some claims (for example a container on a different daemon) cannot be reconciled, `destroy` likewise keeps the stack registered and fails, listing the remaining claims and the `claims.json` path to edit if the original engine is permanently gone. Either way, retry `destroy` once the engine is reachable; there is no `cleanupCommands` field. +`destroy` normally returns `{ runtimeCleanup: "complete" }`. When no owner is running and a new owner cannot start because the stack's container engine reports that its daemon cannot be reached, `destroy` confirms that no owner holds the stack's lease and leaves the registration, port claims and host data untouched, returning `{ runtimeCleanup: "skipped", engine }`. When an owner does run but a labelled container cannot be removed (for example one on a different daemon) or an instance's data cannot be removed, `destroy` likewise keeps the stack registered and fails, listing what remains. Either way, retry `destroy` once the engine is reachable; there is no `cleanupCommands` field. The stack owns database, Functions bootstrap, and command-job directories below its data directory. Storage uploads remain at the caller-supplied Storage `filePath` and are preserved when the stack is destroyed; the caller owns that directory. Host metadata remains under `stateRoot`; native database data uses host files. Docker database data normally uses a managed volume, while existing host data is retained through the host-backed fallback. A host marker records the selected Docker storage and detects a missing or mismatched volume; deleting that volume loses the associated database data. Native snapshot entries live below `cacheRoot`. Docker snapshots share the managed data volume in a separate namespace derived from `cacheRoot`, so they survive source destruction and can use filesystem cloning. A Docker cache hit requires the same daemon, `stateRoot`, and `cacheRoot`. There is no portable tar snapshot API. @@ -89,7 +89,7 @@ The native artifact cache holds one immutable, content-addressed generation per Native PostgreSQL refuses to run as uid 0. When the stack runs as root inside a detected agent sandbox (Claude Code, Modal Sandbox), or `SUPABASE_NATIVE_POSTGRES_USER=` names a non-root system user, only the PostgreSQL process runs as that user: the instance data directory, root key file, socket directory, and the bundle's `pgsodium_getkey.sh` are chowned to it, and the instance directory and its ancestors outside the generation (the artifact cache and stack state roots, which otherwise stay restricted to their owner) receive traverse-only (`o+x`) permission. A directory inside the generation itself is never chmodded; it must already carry that bit from the archive, or the handover fails naming the path. The `pgsodium_getkey.sh` chown is the one accepted exception to generation immutability: the packaged init script hard-codes that cached path and `chmod +x`s it, which only its owner may do. Running as root elsewhere fails before PostgreSQL launches. -Native PostgreSQL listens only on its socket and reads a stack-generated HBA file, written to the socket directory on every launch, instead of `PGDATA/pg_hba.conf`. The socket directory is `pg-` under the validated per-user runtime root, where the digest derives from the data root and instance id, so nothing about it is persisted: launch replaces a leftover, and stop, destroy and orphan recovery remove it. It trusts `supabase_admin`, including through the proxied loopback database port, and requires `scram-sha-256` passwords from every other role. +Native PostgreSQL listens only on its socket and reads a stack-generated HBA file, written to the socket directory on every launch, instead of `PGDATA/pg_hba.conf`. The socket directory is `pg-` under the validated per-user runtime root, where the digest derives from the data root and instance id, so nothing about it is persisted: launch replaces a leftover and stop removes it. Destroying the instance removes it before the instance is unregistered, so a failed removal keeps the instance registered and a retry removes it; orphan recovery of a session stack runs the same destroy. A runtime root that is a symlink, belongs to another user or is writable by others is never trusted: removal warns and leaves its socket directories in place. It trusts `supabase_admin`, including through the proxied loopback database port, and requires `scram-sha-256` passwords from every other role. ### Public ports diff --git a/packages/stack/src/Owner.native-socket-sweep.integration.test.ts b/packages/stack/src/Owner.native-socket-sweep.integration.test.ts index 95204933f6..22bbc97c49 100644 --- a/packages/stack/src/Owner.native-socket-sweep.integration.test.ts +++ b/packages/stack/src/Owner.native-socket-sweep.integration.test.ts @@ -1,7 +1,18 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; import { createHash } from "node:crypto"; -import { Context, Effect, FileSystem, Layer, Path, Redacted, Schema } from "effect"; +import { + Context, + Effect, + Exit, + FileSystem, + Layer, + Path, + PlatformError, + Redacted, + Ref, + Schema, +} from "effect"; import { ChildProcessSpawner } from "effect/unstable/process"; import * as Owner from "./Owner.ts"; import { NativeRuntimeRootBase, nativeRuntimeRootPath } from "./runtime/postgres-user.ts"; @@ -21,7 +32,10 @@ const databaseCreation = Schema.decodeEffect(ServiceCreation)({ }); /** A fresh state namespace, a native runtime root base, and a stack with one saved database. */ -const arrange = Effect.fn("arrange")(function* (id: string) { +const arrange = Effect.fn("arrange")(function* ( + id: string, + lifetime: StackNamespace.SavedStack["lifetime"] = "detached", +) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const root = yield* fs @@ -36,7 +50,7 @@ const arrange = Effect.fn("arrange")(function* (id: string) { runtime: "native", identity: { projectRoot: root, branchContext: "main", stackName: id }, instances: [{ id: "database", creation: yield* databaseCreation }], - lifetime: "detached", + lifetime, composition: { members: [], dependencies: [] }, ports: [], }; @@ -68,64 +82,154 @@ const leaveSocketDirectory = Effect.fn("leaveSocketDirectory")(function* ( return directory; }); -const ownerFor = (arrangement: Arrangement) => - Layer.build( - Owner.layer({ - saved: arrangement.saved, - root: arrangement.dataRoot, - cacheRoot: `${arrangement.root}/cache`, - }).pipe( - Layer.provide( - Layer.merge( - Layer.succeed(StackNamespace.Service, arrangement.state), - Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), - ), +const ownerFor = (arrangement: Arrangement, fileSystem?: Layer.Layer) => { + const owner = Owner.layer({ + saved: arrangement.saved, + root: arrangement.dataRoot, + cacheRoot: `${arrangement.root}/cache`, + }).pipe( + Layer.provide( + Layer.merge( + Layer.succeed(StackNamespace.Service, arrangement.state), + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), ), ), - Layer.provide(Layer.succeed(NativeRuntimeRootBase, arrangement.runtimeRootBase)), ), - ).pipe(Effect.map((context) => Context.get(context, Owner.Service))); + ); + return Layer.build(fileSystem === undefined ? owner : owner.pipe(Layer.provide(fileSystem))).pipe( + Effect.map((context) => Context.get(context, Owner.Service)), + ); +}; + +/** Destroy runs in the caller's context, which names the runtime root base launch would use. */ +const inRuntimeRoot = (arrangement: Arrangement) => + Effect.provideService(NativeRuntimeRootBase, arrangement.runtimeRootBase); const platform = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); -for (const operation of ["stop", "destroy"] as const) - it.live(`${operation} removes the socket directory a killed owner left behind`, () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const arrangement = yield* arrange(`socket-${operation}`); - const leftover = yield* leaveSocketDirectory(arrangement, "database"); - const owner = yield* ownerFor(arrangement); +it.live("destroy removes the socket directory a killed owner left behind", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const arrangement = yield* arrange("socket-destroy"); + const leftover = yield* leaveSocketDirectory(arrangement, "database"); + const owner = yield* ownerFor(arrangement); - yield* owner.namespace[operation]; + yield* owner.namespace.destroy.pipe(inRuntimeRoot(arrangement)); - expect(yield* fs.exists(leftover)).toBe(false); - }), - ).pipe(Effect.provide(platform)), - ); + expect(yield* fs.exists(leftover)).toBe(false); + }), + ).pipe(Effect.provide(platform)), +); + +it.live("a failed socket directory removal keeps the stack registered until a retry succeeds", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const arrangement = yield* arrange("socket-retry"); + const leftover = yield* leaveSocketDirectory(arrangement, "database"); + const failing = yield* Ref.make(true); + const flakyFs = Layer.succeed(FileSystem.FileSystem, { + ...fs, + remove: (target: string, options?: Parameters[1]) => + // The runtime root is resolved to its real path, so match the derived directory name. + target.endsWith(`/${leftover.split("/").at(-1)}`) + ? Ref.get(failing).pipe( + Effect.flatMap((fail) => + fail + ? PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method: "remove", + pathOrDescriptor: target, + cause: new Error("injected removal failure"), + }) + : fs.remove(target, options), + ), + ) + : fs.remove(target, options), + }); + const owner = yield* ownerFor(arrangement, flakyFs); + + const failed = yield* Effect.exit(owner.namespace.destroy.pipe(inRuntimeRoot(arrangement))); + + expect(Exit.isFailure(failed)).toBe(true); + expect(yield* fs.exists(leftover)).toBe(true); + expect(yield* arrangement.state.read(arrangement.saved.id)).toBeDefined(); -it.live("an orphan reclaim removes the socket directory of a stack whose owner is gone", () => + yield* Ref.set(failing, false); + yield* owner.namespace.destroy.pipe(inRuntimeRoot(arrangement)); + + expect(yield* fs.exists(leftover)).toBe(false); + expect(yield* arrangement.state.read(arrangement.saved.id)).toBeUndefined(); + }), + ).pipe(Effect.provide(platform)), +); + +it.live("destroying one database instance removes its socket directory", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const arrangement = yield* arrange("socket-orphan"); + const arrangement = yield* arrange("socket-instance"); const leftover = yield* leaveSocketDirectory(arrangement, "database"); + const owner = yield* ownerFor(arrangement); - const reclaimed = yield* reclaimStack({ - state: arrangement.state, - stateRoot: `${arrangement.root}/state`, - cacheRoot: `${arrangement.root}/cache`, - id: arrangement.saved.id, - }).pipe(Effect.provide(Layer.succeed(NativeRuntimeRootBase, arrangement.runtimeRootBase))); + yield* owner.handlers.destroyService({ id: "database" }).pipe(inRuntimeRoot(arrangement)); - expect(reclaimed).toBe(true); expect(yield* fs.exists(leftover)).toBe(false); }), ).pipe(Effect.provide(platform)), ); +const reclaim = (arrangement: Arrangement) => + reclaimStack({ + state: arrangement.state, + stateRoot: `${arrangement.root}/state`, + cacheRoot: `${arrangement.root}/cache`, + id: arrangement.saved.id, + }).pipe(Effect.provide(Layer.succeed(NativeRuntimeRootBase, arrangement.runtimeRootBase))); + +it.live( + "an orphan reclaim removes the socket directory of a session stack whose owner is gone", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const arrangement = yield* arrange("socket-orphan", "session"); + const leftover = yield* leaveSocketDirectory(arrangement, "database"); + + expect(yield* reclaim(arrangement)).toBe(true); + + expect(yield* fs.exists(leftover)).toBe(false); + }), + ).pipe(Effect.provide(platform)), +); + +it.live( + "an orphan reclaim finds the socket directory of a stack whose data root is a symlink", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const arrangement = yield* arrange("socket-symlink", "session"); + const relocated = `${arrangement.root}/relocated-data`; + yield* fs.makeDirectory(relocated); + yield* fs.symlink(relocated, arrangement.dataRoot); + // Launch hashes the data root's real path, so a leftover is named from it. + const leftover = yield* leaveSocketDirectory( + { ...arrangement, dataRoot: relocated }, + "database", + ); + + expect(yield* reclaim(arrangement)).toBe(true); + + expect(yield* fs.exists(leftover)).toBe(false); + }), + ).pipe(Effect.provide(platform)), +); + it.live("removes only the socket directory of the stack being cleaned up", () => Effect.scoped( Effect.gen(function* () { @@ -140,7 +244,7 @@ it.live("removes only the socket directory of the stack being cleaned up", () => ); const owner = yield* ownerFor(cleaned); - yield* owner.namespace.destroy; + yield* owner.namespace.destroy.pipe(inRuntimeRoot(cleaned)); expect(yield* fs.exists(cleanedLeftover)).toBe(false); expect(yield* fs.exists(siblingLeftover)).toBe(true); @@ -159,7 +263,7 @@ it.live("leaves a socket directory under a runtime root another uid could have t yield* fs.chmod(arrangement.runtimeRoot, 0o777); const owner = yield* ownerFor(arrangement); - yield* owner.namespace.destroy; + yield* owner.namespace.destroy.pipe(inRuntimeRoot(arrangement)); expect(yield* fs.exists(leftover)).toBe(true); }), diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index 68d9a9a23b..f5d50d4884 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -66,7 +66,6 @@ import { import type { CatalogError } from "./services/Recipe.ts"; import * as Container from "./runtime/Container.ts"; import * as Paths from "./namespace/Paths.ts"; -import { removeNativeSocketDirectories } from "./runtime/postgres-user.ts"; import { projectSegmentFor } from "./identity/Identity.ts"; import { stackError, type OwnerRpc } from "./Rpc.ts"; import * as StackNamespace from "./StackNamespace.ts"; @@ -99,23 +98,17 @@ type Handlers = { }; /** - * Removes the native socket directory of every database instance of this stack, then every - * container carrying this stack's identity and data-root labels through the owner's pinned - * engine target. A container that survives removal fails the sweep, so the caller (startup, - * stop, or destroy) can report it and retry instead of proceeding as if the stack were fully - * torn down. + * Removes every container carrying this stack's identity and data-root labels through the + * owner's pinned engine target. A container that survives removal fails the sweep, so the caller + * (startup, stop, or destroy) can report it and retry instead of proceeding as if the stack were + * fully torn down. */ export const sweepContainers = Effect.fn("Owner.sweepContainers")(function* ( - saved: Pick, + saved: Pick, root: string, target: Container.EngineTarget | undefined, ) { const path = yield* Path.Path; - if (saved.runtime === "native") - yield* removeNativeSocketDirectories( - root, - saved.instances.filter(({ creation }) => creation.service === "database").map(({ id }) => id), - ); if (target === undefined) return; const remaining = yield* Container.removeStackContainers({ target, @@ -790,15 +783,9 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ), }; - // The registration's current instances, not the owner's startup snapshot, name the socket - // directories to remove; destroy reads them before it unregisters each instance. - const readCurrent = options.state - .read(stackId) - .pipe(Effect.map((saved) => saved ?? options.saved)); - const sweep = (saved: SavedStack) => - sweepContainers(saved, options.root, options.engineTarget).pipe( - Effect.provideContext(services), - ); + const sweep = sweepContainers(options.saved, options.root, options.engineTarget).pipe( + Effect.provideContext(services), + ); const getStackCredentials = readSaved.pipe( Effect.flatMap(({ credentials }) => credentials === undefined @@ -818,28 +805,24 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { // listener scopes only close afterward, through stop/destroy's ordinary teardown below. stop: network.drain.pipe( Effect.andThen(orchestrator.stopNamespace), - Effect.andThen(readCurrent.pipe(Effect.flatMap(sweep))), + Effect.andThen(sweep), definitionGate.withPermits(1), Effect.withSpan("Owner.stopNamespace"), ), - destroy: readCurrent.pipe( - Effect.flatMap((registered) => - network.beginDestroy.pipe( - // Deferred for the whole of this destroy: every instance's own teardown below closes - // its listeners as usual, but leaves its reservation rows for `releaseStack` to drop - // together, only once destroy is confirmed to leave nothing behind. `ensuring` below - // resumes immediate per-service deletion again on every exit, success, failure, or - // interruption alike, so a service destruction or sweep failure can never leave - // deferral stuck on for a stack that is still otherwise live. - Effect.andThen(network.drain), - Effect.andThen(orchestrator.destroyNamespace), - Effect.andThen(sweep(registered)), - Effect.andThen(removeContainerEnvRoot), - Effect.andThen(network.releaseStack), - Effect.andThen(options.state.remove(stackId)), - Effect.ensuring(network.cancelDestroy), - ), - ), + destroy: network.beginDestroy.pipe( + // Deferred for the whole of this destroy: every instance's own teardown below closes + // its listeners as usual, but leaves its reservation rows for `releaseStack` to drop + // together, only once destroy is confirmed to leave nothing behind. `ensuring` below + // resumes immediate per-service deletion again on every exit, success, failure, or + // interruption alike, so a service destruction or sweep failure can never leave + // deferral stuck on for a stack that is still otherwise live. + Effect.andThen(network.drain), + Effect.andThen(orchestrator.destroyNamespace), + Effect.andThen(sweep), + Effect.andThen(removeContainerEnvRoot), + Effect.andThen(network.releaseStack), + Effect.andThen(options.state.remove(stackId)), + Effect.ensuring(network.cancelDestroy), definitionGate.withPermits(1), Effect.withSpan("Owner.destroyNamespace"), ), diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index e91320a5bf..6efbabfe47 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -543,9 +543,7 @@ export const runStackHost = Effect.fn("StackHost.run")( const saved = yield* state.read(id); if (saved === undefined) return yield* hostError("startup", "Stack is not registered"); const control = yield* bindControl(); - const dataRootPath = path.join(options.stateRoot, saved.id, "data"); - yield* fs.makeDirectory(dataRootPath, { recursive: true }); - const dataRoot = yield* fs.realPath(dataRootPath); + const dataRoot = yield* StackNamespace.resolveStackDataRoot(options.stateRoot, saved.id); const project = projectSegmentFor(saved.identity, path); // Resolved once, here, for this owner's whole lifetime: the container runtime, the // storage helpers, the host-gateway probes and reconcile below all share this one diff --git a/packages/stack/src/StackNamespace.ts b/packages/stack/src/StackNamespace.ts index 0e515e03f2..fa4e51a633 100644 --- a/packages/stack/src/StackNamespace.ts +++ b/packages/stack/src/StackNamespace.ts @@ -6,6 +6,7 @@ import { restrictDirectoryToOwner } from "./runtime/postgres-user.ts"; export { NamespaceError } from "./namespace/Capabilities.ts"; export { LeaseHeldError, LeaseHolder } from "./namespace/Lease.ts"; +export { resolveStackDataRoot } from "./namespace/Paths.ts"; export { PortClaim, SavedStack, diff --git a/packages/stack/src/Sweep.ts b/packages/stack/src/Sweep.ts index ee71312b8c..42a2f1d173 100644 --- a/packages/stack/src/Sweep.ts +++ b/packages/stack/src/Sweep.ts @@ -1,4 +1,4 @@ -import { Context, DateTime, Effect, FileSystem, Layer, Path } from "effect"; +import { Context, DateTime, Effect, Layer } from "effect"; import { ChildProcessSpawner } from "effect/unstable/process"; import { sweepTimeout } from "./HostProcess.ts"; import * as Owner from "./Owner.ts"; @@ -12,9 +12,7 @@ const destroyStack = Effect.fn("Sweep.destroyStack")(function* ( dataRoot: string, cacheRoot: string, ) { - const fs = yield* FileSystem.FileSystem; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - yield* fs.makeDirectory(dataRoot, { recursive: true }); const engineTarget = saved.runtime === "native" ? undefined : yield* resolveEngineTarget(spawner); const context = yield* Layer.build( Owner.layer({ saved, root: dataRoot, cacheRoot, engineTarget }).pipe( @@ -36,8 +34,6 @@ export const reclaimStack = Effect.fn("Sweep.reclaimStack")(function* (options: readonly cacheRoot: string; readonly id: string; }) { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; const { state, id } = options; return yield* Effect.scoped( Effect.gen(function* () { @@ -53,7 +49,7 @@ export const reclaimStack = Effect.fn("Sweep.reclaimStack")(function* (options: }); const saved = yield* state.read(id); if (saved === undefined) return true; - const dataRoot = path.join(yield* fs.realPath(options.stateRoot), id, "data"); + const dataRoot = yield* StackNamespace.resolveStackDataRoot(options.stateRoot, id); if (saved.lifetime === "session") yield* destroyStack(state, saved, dataRoot, options.cacheRoot); else { diff --git a/packages/stack/src/namespace/Paths.ts b/packages/stack/src/namespace/Paths.ts index 77f1ee33e4..37d5de86b7 100644 --- a/packages/stack/src/namespace/Paths.ts +++ b/packages/stack/src/namespace/Paths.ts @@ -66,6 +66,23 @@ export const borrow = Effect.fn("Namespace.Paths.borrow")( }), ); +/** + * The canonical data root of a stack: its `data` directory under `stateRoot`, created if missing + * and resolved through symlinks. Launch and every cleanup path derive container labels and native + * socket directories from this one value, so each reaches the same resources however the state + * root is spelled. + */ +export const resolveStackDataRoot = Effect.fn("Namespace.Paths.resolveStackDataRoot")(function* ( + stateRoot: string, + stackId: string, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dataRoot = path.join(stateRoot, stackId, "data"); + yield* fs.makeDirectory(dataRoot, { recursive: true }); + return yield* fs.realPath(dataRoot); +}); + /** The container mount target for an instance-scoped directory owned on the host. */ export const containerInstancePath = "/instance"; diff --git a/packages/stack/src/runtime/postgres-user.ts b/packages/stack/src/runtime/postgres-user.ts index 915406d445..bf86c88f52 100644 --- a/packages/stack/src/runtime/postgres-user.ts +++ b/packages/stack/src/runtime/postgres-user.ts @@ -251,33 +251,31 @@ export const nativeSocketDirectoryPath = ( ); /** - * Removes the socket directories of a stack's native database instances. A runtime root that is - * missing is nothing to remove; one that is not safe to trust (a symlink, foreign owner or - * writable by others) is warned about and left untouched. + * Removes the socket directory of one native database instance. A runtime root that is missing is + * nothing to remove; one that is not safe to trust (a symlink, foreign owner or writable by others) + * is warned about and left untouched. */ -export const removeNativeSocketDirectories = Effect.fn( - "NativePostgresUser.removeSocketDirectories", -)(function* (dataRoot: string, instanceIds: ReadonlyArray) { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const runtimeRoot = yield* resolveNativeRuntimeRootForRecovery().pipe(Effect.option); - if (Option.isNone(runtimeRoot)) { - const base = yield* NativeRuntimeRootBase; - const leaf = nativeRuntimeRootPath(path, base, process.getuid?.() ?? 0); - if ((yield* lstatPath(leaf)) !== undefined) - yield* Effect.logWarning( - `${leaf} is not a trusted native runtime root; leaving its socket directories`, - ); - return; - } - yield* Effect.annotateCurrentSpan({ "instance.count": instanceIds.length }); - for (const instanceId of instanceIds) +export const removeNativeSocketDirectory = Effect.fn("NativePostgresUser.removeSocketDirectory")( + function* (dataRoot: string, instanceId: string) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const runtimeRoot = yield* resolveNativeRuntimeRootForRecovery().pipe(Effect.option); + if (Option.isNone(runtimeRoot)) { + const base = yield* NativeRuntimeRootBase; + const leaf = nativeRuntimeRootPath(path, base, process.getuid?.() ?? 0); + if ((yield* lstatPath(leaf)) !== undefined) + yield* Effect.logWarning( + `${leaf} is not a trusted native runtime root; leaving its socket directories`, + ); + return; + } yield* fs.remove( yield* nativeSocketDirectoryPath(crypto, path, runtimeRoot.value, dataRoot, instanceId), { recursive: true, force: true }, ); -}); + }, +); /** * Restricts a directory to its owner but keeps an existing traverse-only grant, because a diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index 98652ad338..75745f8b27 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -63,6 +63,7 @@ import type { StackId } from "../identity/StackId.ts"; import { acquireNativeRuntimeRoot, nativeSocketDirectoryPath, + removeNativeSocketDirectory, handOverNativePostgresFiles, openNativePostgresInstance, resolveNativePostgresUser, @@ -934,6 +935,18 @@ export const makeDatabase = ( }), ); + // Removed before the registration that lets recovery recompute its name, so a failure here + // leaves the instance registered to retry. + const removeSocketDirectory = + options.runtime === "native" + ? removeNativeSocketDirectory(options.root, options.instanceId).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(Crypto.Crypto, crypto), + Effect.mapError((cause) => errorFor("destroy", cause)), + ) + : Effect.void; + const definition: ServiceDefinition = { prepare, launch, @@ -943,11 +956,15 @@ export const makeDatabase = ( path, instanceRoot, options.root, - storage === undefined - ? Effect.void - : storage - .destroyData(postgresVersion(context.config.version)) - .pipe(Effect.mapError((cause) => errorFor("destroy", cause))), + removeSocketDirectory.pipe( + Effect.andThen( + storage === undefined + ? Effect.void + : storage + .destroyData(postgresVersion(context.config.version)) + .pipe(Effect.mapError((cause) => errorFor("destroy", cause))), + ), + ), errorFor, ), }; From 01b0de945d8207d8282c0b8d1453cabe343740cd Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 01:12:23 +0200 Subject: [PATCH 064/142] chore(stack): drop lint exceptions for removed claims consumers --- .oxlintrc.json | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/.oxlintrc.json b/.oxlintrc.json index a2d52a5fc0..0d3ae82fba 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -45,7 +45,7 @@ } }, { - // The namespace owns claims, paths, leases, artifact pins, and port reservations; everywhere + // The namespace owns paths, leases, artifact pins, and port reservations; everywhere // in the repo, outside `src/namespace/**` itself, reaches that state through // `StackNamespace.ts`, not by importing namespace internals directly. Repo-wide (apps/cli // included) so a deep or relative import of a namespace module fails from any package; the @@ -82,7 +82,6 @@ "packages/stack/src/Ports.integration.test.ts", "packages/stack/src/Owner.integration.test.ts", "packages/stack/src/port-reservation.e2e.test.ts", - "packages/stack/src/StackHost.ts", "packages/stack/src/host/CommandRunner.ts", "packages/stack/src/preparation/ArtifactStore.ts", "packages/stack/src/preparation/artifacts.integration.test.ts", @@ -95,10 +94,7 @@ "packages/stack/src/services/Database.ts", "packages/stack/src/services/DatabaseSnapshot.ts", "packages/stack/src/services/ProcessRecipe.ts", - "packages/stack/src/services/Recipe.ts", - "packages/stack/src/storage/DockerDatabaseStorage.ts", - "packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts", - "packages/stack/tests/claims.ts" + "packages/stack/src/storage/DockerDatabaseStorage.ts" ], "rules": { "no-restricted-imports": [ @@ -250,18 +246,14 @@ // none of them touch raw node:fs/node:sqlite themselves, so that restriction stays on. // - Network.ts, Ports.ts (+its integration test), port-reservation.e2e.test.ts: read/reserve // ports through PortReservations, not yet on the facade. - // - Owner.ts: needs Claims, namespaceError, no-follow lstatPath, and owned/borrowed Paths. - // - StackHost.ts: reads Claims. + // - Owner.ts: needs PortReservations and owned/borrowed Paths. // - ArtifactStore.ts, artifacts.integration.test.ts: use the sqlite-pin driver and Pin; // artifact pins are not on the facade yet. // - Container.ts, Database.ts, ProcessRecipe.ts, Catalog.ts: need owned/borrowed Paths // and/or Environment. // - NativeProcess.ts, native-process.integration.test.ts: need Environment. // - postgres-user.ts: needs the no-follow lstatPath. - // - Recipe.ts: needs Claims types only. - // - DockerDatabaseStorage.ts (+its integration test): needs Claims types and Publication. - // - tests/claims.ts: a no-op `ContainerClaims`/`DirectoryClaims` fixture for tests that don't - // exercise the namespace's own reconcile loop. + // - DockerDatabaseStorage.ts: needs Publication. "files": [ "packages/stack/src/Network.ts", "packages/stack/src/Owner.ts", @@ -269,7 +261,6 @@ "packages/stack/src/Ports.integration.test.ts", "packages/stack/src/Owner.integration.test.ts", "packages/stack/src/port-reservation.e2e.test.ts", - "packages/stack/src/StackHost.ts", "packages/stack/src/preparation/ArtifactStore.ts", "packages/stack/src/preparation/artifacts.integration.test.ts", "packages/stack/src/runtime/Container.ts", @@ -279,10 +270,7 @@ "packages/stack/src/services/Catalog.ts", "packages/stack/src/services/Database.ts", "packages/stack/src/services/ProcessRecipe.ts", - "packages/stack/src/services/Recipe.ts", - "packages/stack/src/storage/DockerDatabaseStorage.ts", - "packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts", - "packages/stack/tests/claims.ts" + "packages/stack/src/storage/DockerDatabaseStorage.ts" ], "rules": { "no-restricted-imports": [ @@ -313,7 +301,7 @@ } }, { - // `host/CommandRunner.ts` needs Claims, Environment, and borrow from the namespace, plus the + // `host/CommandRunner.ts` needs Environment and borrow from the namespace, plus the // raw `rmdir` Effect's FileSystem has no non-recursive equivalent for (see its existing // disable comment); sqlite stays off-limits. "files": ["packages/stack/src/host/CommandRunner.ts"], From 18c9365980a6023650d2c3855ce5537a11704ff7 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 01:20:02 +0200 Subject: [PATCH 065/142] refactor(stack): reject pinned public ports in the native range instead of probing reservations --- .../experimental/stack/start/SIDE_EFFECTS.md | 2 +- packages/stack/README.md | 6 +-- packages/stack/src/Owner.ts | 16 ------ packages/stack/src/Ports.integration.test.ts | 51 ++----------------- packages/stack/src/Ports.ts | 32 +++++------- .../stack/src/namespace/PortReservations.ts | 13 +---- .../stack/src/port-reservation.e2e.test.ts | 30 +++++++++++ ...nalyticsVectorImgproxy.integration.test.ts | 2 - .../services/AuthStorage.integration.test.ts | 2 - .../src/services/Catalog.integration.test.ts | 2 - packages/stack/src/services/Catalog.ts | 1 - .../services/Functions.integration.test.ts | 9 ---- .../src/services/Mail.integration.test.ts | 2 - .../src/services/Pooler.integration.test.ts | 2 - .../ProcessRecipe.integration.test.ts | 18 ------- packages/stack/src/services/ProcessRecipe.ts | 10 +--- .../src/services/Realtime.integration.test.ts | 2 - packages/stack/src/services/Recipe.ts | 3 -- .../src/services/Rest.integration.test.ts | 2 - .../src/services/Studio.integration.test.ts | 2 - .../src/services/Vector.integration.test.ts | 2 - packages/stack/tests/port-reservations.ts | 7 --- 22 files changed, 54 insertions(+), 162 deletions(-) delete mode 100644 packages/stack/tests/port-reservations.ts diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index a6799883c5..d0277d3aa5 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -47,7 +47,7 @@ different port; only automatic allocation tries another candidate. On macOS a pr a narrow race against the pre-bind probe before a brand-new listener exists. Native backend ports (not publicly exposed) are reserved from 10000–19999, disjoint from the public auto range and below the OS ephemeral range on every supported platform; this range is not -configurable, and a host whose ephemeral range has been widened to overlap it reintroduces the +configurable, and a public port pinned inside it is rejected. A host whose ephemeral range has been widened to overlap it reintroduces the ephemeral-port race this reservation exists to avoid. For a new stack, `--runtime auto` selects Docker when `docker version` reaches its daemon, then diff --git a/packages/stack/README.md b/packages/stack/README.md index 7e733a2de0..151c0927c1 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -115,9 +115,9 @@ unmounted or otherwise temporarily unreachable state root looks identical to a d be reclaimed the same way. The registry does not reserve across OS users; isolation between users' stacks still relies only on the kernel refusing a second bind. A native backend's own (never public) ports are reserved directly from 10000–19999, disjoint from the public auto range and below the OS -ephemeral range on every supported platform; this is not configurable, and a host whose ephemeral -range has been widened to overlap 10000–19999 reintroduces the ephemeral-port race this reservation -exists to avoid. +ephemeral range on every supported platform; this is not configurable, and a public port pinned +inside it is rejected. A host whose ephemeral range has been widened to overlap 10000–19999 +reintroduces the ephemeral-port race this reservation exists to avoid. ## Composition and operation scope diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index f5d50d4884..68942ca9db 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -22,8 +22,6 @@ import type { Rpc, RpcGroup } from "effect/unstable/rpc"; import { failureMessage } from "./internal/failure-message.ts"; import * as Network from "./Network.ts"; import type { NetworkEndpoint, NetworkNamespace } from "./Network.ts"; -import * as PortReservations from "./namespace/PortReservations.ts"; -import { PortError } from "./Ports.ts"; import * as Orchestrator from "./Orchestrator.ts"; import type { CompositionConfig } from "./Orchestrator.ts"; import { @@ -270,7 +268,6 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { const fs = Context.get(services, FileSystem.FileSystem); const spawner = Context.get(services, ChildProcessSpawner.ChildProcessSpawner); const network = yield* Network.Service; - const portReservations = yield* PortReservations.Service; // A native stack has no engine to lose, so every cleanup failure is retried forever; a docker // stack probes the pinned target directly, never by classifying a failure's cause. const engineAvailable: Effect.Effect = @@ -283,17 +280,6 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { const removeContainerEnvRoot = fs .remove(path.join(options.root, Paths.CONTAINER_ENV_DIRNAME), { recursive: true, force: true }) .pipe(Effect.mapError(serviceError("cleanup"))); - const isPubliclyReserved = (port: number) => - portReservations.isReserved(port).pipe( - Effect.mapError( - (cause) => - new PortError({ - key: "native", - message: "Unable to check the public port reservation registry", - cause, - }), - ), - ); const rejectWhileDraining = (options.draining ?? Effect.succeed(false)).pipe( Effect.flatMap((isDraining) => isDraining @@ -436,7 +422,6 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { cacheRoot: options.cacheRoot, runtime, helpers, - isPubliclyReserved, ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), ...(options.engineTarget === undefined ? {} : { engineTarget: options.engineTarget }), }).pipe(Effect.provideContext(services)); @@ -917,5 +902,4 @@ export const layer = (options: Omit) => runtime: options.saved.runtime, }), ), - Layer.provide(PortReservations.layer), ); diff --git a/packages/stack/src/Ports.integration.test.ts b/packages/stack/src/Ports.integration.test.ts index 7f25777bde..8f2b6a82cf 100644 --- a/packages/stack/src/Ports.integration.test.ts +++ b/packages/stack/src/Ports.integration.test.ts @@ -62,9 +62,6 @@ const saveStack = (state: StackNamespace.Interface, root: string, id: string) => const probedBind = (key: string) => (host: string, port: number) => probeVacant(process.platform)(key, host, port).pipe(Effect.andThen(bindTcp(host, port))); -/** Stands in for the real per-user registry check in tests that are not exercising it. */ -const neverPubliclyReserved = () => Effect.succeed(false); - /** * Binds a real wildcard listener directly in the native span, skipping any already-occupied * candidate, instead of reserving a port through the allocator and racing to rebind it as a @@ -350,7 +347,7 @@ it.live("reserveNativePort skips a port a wildcard listener already holds", () = // past the wildcard listener instead of merely missing it by chance. const key = keyWithFirstCandidate(blocked.port); - const reserved = yield* reserveNativePort(key, new Set(), neverPubliclyReserved); + const reserved = yield* reserveNativePort(key, new Set()); expect(reserved.port).not.toBe(blocked.port); }), ).pipe(Effect.provide(NodeServices.layer)), @@ -361,55 +358,13 @@ it.live("reserveNativePort excludes a port a previous attempt lost from the next Effect.gen(function* () { const key = `native-exclude-${randomUUID()}`; const firstScope = yield* Scope.make(); - const first = yield* reserveNativePort(key, new Set(), neverPubliclyReserved).pipe( + const first = yield* reserveNativePort(key, new Set()).pipe( Effect.provideService(Scope.Scope, firstScope), ); yield* Scope.close(firstScope, Exit.void); - const second = yield* reserveNativePort(key, new Set([first.port]), neverPubliclyReserved); + const second = yield* reserveNativePort(key, new Set([first.port])); expect(second.port).not.toBe(first.port); }), ).pipe(Effect.provide(NodeServices.layer)), ); - -it.live( - "reserveNativePort skips a stopped stack's fixed public reservation inside the native range", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const id = `native-public-${randomUUID()}`; - const state = yield* makeTestState(root); - yield* saveStack(state, root, id); - const portReservations = yield* PortReservations.Service; - const stateRoot = yield* fs.realPath(root); - - // A free candidate in the native span, claimed as stack `id`'s durable fixed public - // reservation with no live listener: the stopped-but-not-destroyed state the registry - // keeps. The row, not a socket, is what the native allocator must respect here. - const target = yield* bindNativeWildcard(); - yield* Effect.callback((resume) => { - target.server.close(() => resume(Effect.void)); - }); - const conflict = yield* portReservations.reserve(stateRoot, id, "fixed", target.port); - yield* Effect.addFinalizer(() => - portReservations.release(stateRoot, id, "fixed").pipe(Effect.orDie), - ); - expect(conflict).toBeUndefined(); - - const key = keyWithFirstCandidate(target.port); - const isPubliclyReserved = (port: number) => - portReservations.isReserved(port).pipe(Effect.orDie); - const reserved = yield* reserveNativePort(key, new Set(), isPubliclyReserved); - expect(reserved.port).not.toBe(target.port); - }), - ).pipe( - Effect.provide( - Layer.merge( - NodeServices.layer, - PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), - ), - ), - ), -); diff --git a/packages/stack/src/Ports.ts b/packages/stack/src/Ports.ts index 0e5fa10298..e2239aa658 100644 --- a/packages/stack/src/Ports.ts +++ b/packages/stack/src/Ports.ts @@ -132,8 +132,9 @@ export const probeVacant = /** * Disjoint from the public auto range (`portBase`..`portBase + portSpan`) and contiguous below it, - * so a native backend's direct bind can never land on a port the per-user registry is reserving for - * a public listener. Both ranges stay below the OS ephemeral range (ADR 0017). + * and a pinned public port inside it is rejected, so a native backend's direct bind can never land + * on a port the per-user registry is reserving for a public listener. Both ranges stay below the + * OS ephemeral range (ADR 0017). */ export const nativePortBase = 10000; export const nativePortSpan = portBase - nativePortBase; @@ -185,19 +186,15 @@ const bindNativePort = ( /** * Reserves a native backend's private port, hash-seeding the scan from `key` so a reopened * instance starts from the same candidate while separate keys spread out (mirrors the public auto - * scan's hash seed). Skips a candidate `isPubliclyReserved` reports, so a native bind can never - * take a port the per-user registry holds for a stopped stack's public listener (that stack owns - * no live socket to probe or bind against). Otherwise probes with {@link probeVacant} before the - * real bind, since a loopback-only bind can silently coexist with a wildcard listener on macOS, - * BSD, and Windows. A port a prior attempt in this batch lost is passed in `excluded` so a retry - * advances instead of repeating it. Never saved: backend ports are private and are not stable - * across restarts. + * scan's hash seed). Probes with {@link probeVacant} before the real bind, since a loopback-only + * bind can silently coexist with a wildcard listener on macOS, BSD, and Windows. A port a prior + * attempt in this batch lost is passed in `excluded` so a retry advances instead of repeating it. + * Never saved: backend ports are private and are not stable across restarts. */ export const reserveNativePort = Effect.fn("Ports.reserveNativePort")( ( key: string, excluded: ReadonlySet, - isPubliclyReserved: (port: number) => Effect.Effect, platform: NodeJS.Platform = process.platform, ): Effect.Effect => Effect.acquireRelease( @@ -209,14 +206,6 @@ export const reserveNativePort = Effect.fn("Ports.reserveNativePort")( for (let attempt = 0; attempt < nativePortSpan && failures < 64; attempt++) { const port = nativePortBase + ((start + attempt * nativePortStride) % nativePortSpan); if (excluded.has(port)) continue; - if (yield* isPubliclyReserved(port)) { - failures++; - lastFailure = new PortError({ - key, - message: `Port ${port} is reserved by a public listener`, - }); - continue; - } const probed = yield* Effect.exit(probe(key, "127.0.0.1", port)); if (Exit.isFailure(probed)) { const probeError = Cause.findErrorOption(probed.cause); @@ -273,6 +262,13 @@ const resolveRequest = ( if (requested === "auto") return Effect.succeed({ saved, requested }); if (!Number.isInteger(requested) || requested < 1 || requested > 65535) return Effect.fail(new PortError({ key: request.key, message: "Invalid public port" })); + if (requested >= nativePortBase && requested < portBase) + return Effect.fail( + new PortError({ + key: request.key, + message: `Public port ${requested} is inside ${nativePortBase}-${portBase - 1}, which is reserved for native service ports; choose a port outside that range`, + }), + ); return Effect.succeed({ saved, requested }); }; diff --git a/packages/stack/src/namespace/PortReservations.ts b/packages/stack/src/namespace/PortReservations.ts index 8919061c8d..05c5f15c55 100644 --- a/packages/stack/src/namespace/PortReservations.ts +++ b/packages/stack/src/namespace/PortReservations.ts @@ -22,8 +22,6 @@ export interface Interface { stackId: string, endpoint: string, ) => Effect.Effect; - /** True when any stack, at any state root, holds a public reservation for `port`. */ - readonly isReserved: (port: number) => Effect.Effect; /** * Reserves `port` for `(stateRoot, stackId, endpoint)`. Resolves to `undefined` once the row is * committed, or the live {@link Holder} already occupying `port` when it is taken. @@ -242,15 +240,6 @@ const make = Effect.fn("PortReservations.make")(function* (): Effect.fn.Return< return isHolderRow(row) ? row : undefined; }; - // Untraced: the native backend scan in `Ports.ts` calls this once per candidate port, and its - // own span already records the attempt and failure counts for the whole scan. - const isReserved = Effect.fnUntraced(function* (port: number) { - return yield* Effect.try({ - try: () => holderOf(port) !== undefined, - catch: (cause) => namespaceError("find", cause), - }); - }); - // Untraced: `Ports.ts`'s auto scan calls `reserve` and `reclaim` once per candidate port, and its // own span already records the attempt and failure counts for the whole scan. const reserve = Effect.fnUntraced(function* ( @@ -362,7 +351,7 @@ const make = Effect.fn("PortReservations.make")(function* (): Effect.fn.Return< }); }); - return { find, isReserved, reserve, reclaim, release, releaseStack }; + return { find, reserve, reclaim, release, releaseStack }; }); export const layer = Layer.effect(Service, make().pipe(Effect.map(Service.of))); diff --git a/packages/stack/src/port-reservation.e2e.test.ts b/packages/stack/src/port-reservation.e2e.test.ts index a4e7452951..e9ef14ab88 100644 --- a/packages/stack/src/port-reservation.e2e.test.ts +++ b/packages/stack/src/port-reservation.e2e.test.ts @@ -462,3 +462,33 @@ it.live( ), { timeout: 120_000 }, ); + +it.live( + "pinning a public port inside the native range fails naming the range and reserves nothing", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-native-range-cache-", + }); + const { stack, stateRoot } = yield* makeStack(fs, cacheRoot); + const mail = yield* stack.services.create(mailOn(15_432)); + + const failure = yield* Effect.flip(mail.start); + + expect(failure.message).toContain("10000-19999"); + expect(failure.message).toContain("choose a port outside that range"); + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: stateRoot })), + ); + expect((yield* state.read(stack.id))?.ports).toEqual([]); + const reservationContext = yield* Layer.build(PortReservations.layer); + const portReservations = Context.get(reservationContext, PortReservations.Service); + expect( + yield* portReservations.find(yield* fs.realPath(stateRoot), stack.id, `${mail.id}:http`), + ).toBeUndefined(); + }), + ), + { timeout: 120_000 }, +); diff --git a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts index 274f5df502..18b3a92d1a 100644 --- a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts +++ b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts @@ -7,7 +7,6 @@ import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; -import { noPublicPortReservations } from "../../tests/port-reservations.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ @@ -16,7 +15,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - isPubliclyReserved: noPublicPortReservations, }); const dockerOptions = (root: string) => ({ diff --git a/packages/stack/src/services/AuthStorage.integration.test.ts b/packages/stack/src/services/AuthStorage.integration.test.ts index 7d25742d24..871efe661c 100644 --- a/packages/stack/src/services/AuthStorage.integration.test.ts +++ b/packages/stack/src/services/AuthStorage.integration.test.ts @@ -8,7 +8,6 @@ import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; -import { noPublicPortReservations } from "../../tests/port-reservations.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ @@ -17,7 +16,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - isPubliclyReserved: noPublicPortReservations, }); const dockerOptions = (root: string) => ({ diff --git a/packages/stack/src/services/Catalog.integration.test.ts b/packages/stack/src/services/Catalog.integration.test.ts index 0e0165b297..be06f948a4 100644 --- a/packages/stack/src/services/Catalog.integration.test.ts +++ b/packages/stack/src/services/Catalog.integration.test.ts @@ -2,7 +2,6 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Effect, Exit, FileSystem, Layer, Redacted } from "effect"; import { makeServiceRecipe } from "./Catalog.ts"; -import { noPublicPortReservations } from "../../tests/port-reservations.ts"; const options = (root: string) => ({ stackId: "catalog-recipe", @@ -10,7 +9,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - isPubliclyReserved: noPublicPortReservations, }); describe("service catalog", () => { diff --git a/packages/stack/src/services/Catalog.ts b/packages/stack/src/services/Catalog.ts index 7924379422..b551d4161c 100644 --- a/packages/stack/src/services/Catalog.ts +++ b/packages/stack/src/services/Catalog.ts @@ -374,7 +374,6 @@ export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( spawner, container, borrowCallerPath, - isPubliclyReserved: options.isPubliclyReserved, }; switch (creation.service) { case "rest": diff --git a/packages/stack/src/services/Functions.integration.test.ts b/packages/stack/src/services/Functions.integration.test.ts index a9b9ccee6e..7ed37b2529 100644 --- a/packages/stack/src/services/Functions.integration.test.ts +++ b/packages/stack/src/services/Functions.integration.test.ts @@ -22,7 +22,6 @@ import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import * as Functions from "./Functions.ts"; import { makeProcessRecipe } from "./ProcessRecipe.ts"; -import { noPublicPortReservations } from "../../tests/port-reservations.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; @@ -103,7 +102,6 @@ describe("service catalog", () => { }, }, { - isPubliclyReserved: noPublicPortReservations, ...dockerOptions(root), stackId, instanceId, @@ -166,7 +164,6 @@ describe("service catalog", () => { config: { functionsRoot, databaseUrl: "postgres://functions-db" }, }, { - isPubliclyReserved: noPublicPortReservations, ...options(root), stackId, instanceId, @@ -260,7 +257,6 @@ describe("service catalog", () => { }, }, { - isPubliclyReserved: noPublicPortReservations, ...options(stackRoot), stackId: "d".repeat(64), instanceId: "ancestor", @@ -334,7 +330,6 @@ describe("service catalog", () => { }, }, { - isPubliclyReserved: noPublicPortReservations, ...options(stackRoot), stackId: "e".repeat(64), instanceId: "deno-config", @@ -409,7 +404,6 @@ describe("service catalog", () => { }, }, { - isPubliclyReserved: noPublicPortReservations, ...options(stackRoot), stackId: "f".repeat(64), instanceId: "plain-deno-config", @@ -523,7 +517,6 @@ for (const runtime of ["native", "docker"] as const) { }, }, { - isPubliclyReserved: noPublicPortReservations, ...options(stackRoot), stackId: "c".repeat(64), instanceId: "configured", @@ -634,7 +627,6 @@ it.effect("passes POSIX project paths to a docker Functions container from a Win }; const deps = { borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), - isPubliclyReserved: noPublicPortReservations, fs: yield* FileSystem.FileSystem, path: yield* Path.Path.pipe(Effect.provide(NodePath.layerWin32)), crypto: yield* Crypto.Crypto, @@ -645,7 +637,6 @@ it.effect("passes POSIX project paths to a docker Functions container from a Win const recipe = yield* makeProcessRecipe( creation, { - isPubliclyReserved: noPublicPortReservations, stackId: "e".repeat(64), instanceId: "windows", root: "C:\\Users\\dev\\stack", diff --git a/packages/stack/src/services/Mail.integration.test.ts b/packages/stack/src/services/Mail.integration.test.ts index db6ee37ca1..6f320190a0 100644 --- a/packages/stack/src/services/Mail.integration.test.ts +++ b/packages/stack/src/services/Mail.integration.test.ts @@ -4,7 +4,6 @@ import { Effect, Exit, FileSystem, Layer, Path } from "effect"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; -import { noPublicPortReservations } from "../../tests/port-reservations.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; @@ -14,7 +13,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - isPubliclyReserved: noPublicPortReservations, }); const dockerOptions = (root: string) => ({ diff --git a/packages/stack/src/services/Pooler.integration.test.ts b/packages/stack/src/services/Pooler.integration.test.ts index 02a621a407..b155189bcb 100644 --- a/packages/stack/src/services/Pooler.integration.test.ts +++ b/packages/stack/src/services/Pooler.integration.test.ts @@ -7,7 +7,6 @@ import { ProxyError } from "../Proxy.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { noPublicPortReservations } from "../../tests/port-reservations.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; const options = (root: string) => ({ @@ -16,7 +15,6 @@ const options = (root: string) => ({ root, cacheRoot: "/tmp/supabase-stack-artifacts", runtime: "native" as const, - isPubliclyReserved: noPublicPortReservations, }); const dockerOptions = (root: string) => ({ diff --git a/packages/stack/src/services/ProcessRecipe.integration.test.ts b/packages/stack/src/services/ProcessRecipe.integration.test.ts index 3ada73f3e9..3642f6f2a2 100644 --- a/packages/stack/src/services/ProcessRecipe.integration.test.ts +++ b/packages/stack/src/services/ProcessRecipe.integration.test.ts @@ -49,7 +49,6 @@ import { import type { CatalogOptions, RecipeCreation } from "./Recipe.ts"; import * as Realtime from "./Realtime.ts"; import * as Pooler from "./Pooler.ts"; -import { noPublicPortReservations } from "../../tests/port-reservations.ts"; type TestCreation = RecipeCreation<"rest", Record> & { readonly service: "rest"; @@ -77,7 +76,6 @@ const options: CatalogOptions = { root: "/tmp/process-recipe-test", cacheRoot: "/tmp/process-recipe-test-cache", runtime: "docker", - isPubliclyReserved: noPublicPortReservations, }; const spec: ProcessRecipeSpec = { @@ -194,7 +192,6 @@ describe("ProcessRecipe launch cleanup", () => { spawner: yield* ChildProcessSpawner.ChildProcessSpawner, container, borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), - isPubliclyReserved: noPublicPortReservations, } satisfies ProcessDependencies; const recipe = yield* makeProcessRecipe(creation, options, dependencies, spec); const service = yield* makeStandaloneService(recipe.definition, { @@ -283,7 +280,6 @@ describe("ProcessRecipe launch cleanup", () => { spawner, container: undefined, borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), - isPubliclyReserved: noPublicPortReservations, } satisfies ProcessDependencies; const recipe = yield* makeProcessRecipe(creation, nativeOptions, dependencies, nativeSpec); const service = yield* makeStandaloneService(recipe.definition, { @@ -395,7 +391,6 @@ describe("ProcessRecipe launch cleanup", () => { spawner, container: undefined, borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), - isPubliclyReserved: noPublicPortReservations, } satisfies ProcessDependencies; const recipe = yield* makeProcessRecipe(creation, nativeOptions, dependencies, nativeSpec); const service = yield* makeStandaloneService(recipe.definition, { @@ -460,7 +455,6 @@ const realtimeService = Effect.fn(function* (container: ContainerRuntime) { const recipe = yield* makeProcessRecipe( creation, { - isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-test", instanceId: "instance", root: "/unused", @@ -469,7 +463,6 @@ const realtimeService = Effect.fn(function* (container: ContainerRuntime) { }, { borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), - isPubliclyReserved: noPublicPortReservations, fs: yield* FileSystem.FileSystem, path: yield* Path.Path, crypto: yield* Crypto.Crypto, @@ -679,7 +672,6 @@ const nativeRestRecipe = Effect.fn(function* ( }, { borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), - isPubliclyReserved: noPublicPortReservations, fs, path, crypto, @@ -957,7 +949,6 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { - isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-port-race", instanceId: "instance", root, @@ -967,7 +958,6 @@ describe("process recipe startup", () => { }, { borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), - isPubliclyReserved: noPublicPortReservations, fs, path, crypto, @@ -1036,7 +1026,6 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { - isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-port-exhaustion", instanceId: "instance", root, @@ -1046,7 +1035,6 @@ describe("process recipe startup", () => { }, { borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), - isPubliclyReserved: noPublicPortReservations, fs, path, crypto, @@ -1176,7 +1164,6 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { - isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-deadline", instanceId: "instance", root, @@ -1186,7 +1173,6 @@ describe("process recipe startup", () => { }, { borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), - isPubliclyReserved: noPublicPortReservations, fs, path, crypto, @@ -1241,7 +1227,6 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { - isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-unrelated-failure", instanceId: "instance", root, @@ -1251,7 +1236,6 @@ describe("process recipe startup", () => { }, { borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), - isPubliclyReserved: noPublicPortReservations, fs, path, crypto, @@ -1348,7 +1332,6 @@ describe("process recipe startup", () => { const recipe = yield* makeProcessRecipe( creation, { - isPubliclyReserved: noPublicPortReservations, stackId: "process-recipe-exit-failure", instanceId: "instance", root, @@ -1358,7 +1341,6 @@ describe("process recipe startup", () => { }, { borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), - isPubliclyReserved: noPublicPortReservations, fs, path, crypto, diff --git a/packages/stack/src/services/ProcessRecipe.ts b/packages/stack/src/services/ProcessRecipe.ts index 7a5f02728c..e523adbbca 100644 --- a/packages/stack/src/services/ProcessRecipe.ts +++ b/packages/stack/src/services/ProcessRecipe.ts @@ -28,7 +28,7 @@ import { } from "../Artifacts.ts"; import * as Environment from "../namespace/Environment.ts"; import { containerInstancePath, destroyOwnedRoot, type BorrowedPath } from "../namespace/Paths.ts"; -import { accepts, reserveNativePort, type PortError } from "../Ports.ts"; +import { accepts, reserveNativePort } from "../Ports.ts"; import { type ContainerError, type ContainerProcess, @@ -186,8 +186,6 @@ export interface ProcessDependencies { readonly container: ContainerRuntime | undefined; /** Validates a caller-supplied path against the stack's data root; see `namespace/Paths.borrow`. */ readonly borrowCallerPath: (candidate: string) => Effect.Effect; - /** Read-only: true when the per-user registry holds `port` for any stack's public listener. */ - readonly isPubliclyReserved: (port: number) => Effect.Effect; } const serviceError = mapToServiceError; @@ -505,11 +503,7 @@ export const makeProcessRecipe = const reservations = yield* Effect.forEach( portNames, ([name]) => - reserveNativePort( - keyFor(name), - excluded.get(keyFor(name)) ?? emptyNativePortSet, - deps.isPubliclyReserved, - ), + reserveNativePort(keyFor(name), excluded.get(keyFor(name)) ?? emptyNativePortSet), { concurrency: 1 }, ).pipe( Scope.provide(portScope), diff --git a/packages/stack/src/services/Realtime.integration.test.ts b/packages/stack/src/services/Realtime.integration.test.ts index 03fba42f29..ae9d5f9db6 100644 --- a/packages/stack/src/services/Realtime.integration.test.ts +++ b/packages/stack/src/services/Realtime.integration.test.ts @@ -6,7 +6,6 @@ import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { noPublicPortReservations } from "../../tests/port-reservations.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; @@ -16,7 +15,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - isPubliclyReserved: noPublicPortReservations, }); const dockerOptions = (root: string) => ({ diff --git a/packages/stack/src/services/Recipe.ts b/packages/stack/src/services/Recipe.ts index b3c0f02191..b5b975496e 100644 --- a/packages/stack/src/services/Recipe.ts +++ b/packages/stack/src/services/Recipe.ts @@ -3,7 +3,6 @@ import type { Stream } from "effect"; import type { Effect, Ref } from "effect"; import type { ServiceKind } from "../Artifacts.ts"; import type { ServiceDefinition } from "../Service.ts"; -import type { PortError } from "../Ports.ts"; import type { EngineTarget, HostGateway } from "../runtime/Container.ts"; import type { DockerHelperRegistry } from "../storage/DockerHelperRegistry.ts"; @@ -57,8 +56,6 @@ export interface CatalogOptions { readonly hostGateway?: HostGateway; /** The engine endpoint and identity the owner resolved once at startup; absent when native. */ readonly engineTarget?: EngineTarget; - /** Read-only: true when the per-user registry holds `port` for any stack's public listener. */ - readonly isPubliclyReserved: (port: number) => Effect.Effect; } export interface CatalogLog { diff --git a/packages/stack/src/services/Rest.integration.test.ts b/packages/stack/src/services/Rest.integration.test.ts index 0e4bfc97c3..eef28b0cad 100644 --- a/packages/stack/src/services/Rest.integration.test.ts +++ b/packages/stack/src/services/Rest.integration.test.ts @@ -11,7 +11,6 @@ import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { ProxyError } from "../Proxy.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { noPublicPortReservations } from "../../tests/port-reservations.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; @@ -37,7 +36,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - isPubliclyReserved: noPublicPortReservations, }); const dockerOptions = (root: string) => ({ diff --git a/packages/stack/src/services/Studio.integration.test.ts b/packages/stack/src/services/Studio.integration.test.ts index 8a26fdad59..29d665162b 100644 --- a/packages/stack/src/services/Studio.integration.test.ts +++ b/packages/stack/src/services/Studio.integration.test.ts @@ -3,7 +3,6 @@ import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Layer } from "effect"; import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; -import { noPublicPortReservations } from "../../tests/port-reservations.ts"; const options = (root: string) => ({ stackId: "catalog-studio", @@ -11,7 +10,6 @@ const options = (root: string) => ({ root, cacheRoot: `${root}/cache`, runtime: "native" as const, - isPubliclyReserved: noPublicPortReservations, }); describe("studio recipe", () => { diff --git a/packages/stack/src/services/Vector.integration.test.ts b/packages/stack/src/services/Vector.integration.test.ts index ebf2215ffb..aaddd903fd 100644 --- a/packages/stack/src/services/Vector.integration.test.ts +++ b/packages/stack/src/services/Vector.integration.test.ts @@ -5,7 +5,6 @@ import { Effect, FileSystem, Layer } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; -import { noPublicPortReservations } from "../../tests/port-reservations.ts"; import { dockerEngineTarget } from "../../tests/engine-target.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; @@ -16,7 +15,6 @@ const options = (root: string, runtime: "docker" | "native") => ({ cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, runtime, ...(runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), - isPubliclyReserved: noPublicPortReservations, }); describe("vector recipe", () => { diff --git a/packages/stack/tests/port-reservations.ts b/packages/stack/tests/port-reservations.ts deleted file mode 100644 index 8cc756415a..0000000000 --- a/packages/stack/tests/port-reservations.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { Effect } from "effect"; -import type { PortError } from "../src/Ports.ts"; - -/** Stands in for the real per-user registry check in tests not exercising it (covered separately - * by `Ports.integration.test.ts`). */ -export const noPublicPortReservations = (_port: number): Effect.Effect => - Effect.succeed(false); From 7a6a5eea82740c2166501206f5a0919370bdeeaf Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 01:41:07 +0200 Subject: [PATCH 066/142] refactor(stack): keep port assignments only in the reservation registry --- .oxlintrc.json | 2 + .../db-config.integration.test.ts | 1 - .../commands/db/dump/dump.integration.test.ts | 1 - .../db/reset/reset.integration.test.ts | 1 - .../generate/generate.integration.test.ts | 1 - .../declarative/sync/sync.integration.test.ts | 1 - .../db/start/start.integration.test.ts | 1 - .../start-export-pointer.integration.test.ts | 1 - .../stack/status/status.integration.test.ts | 1 - .../serve/serve.stack.integration.test.ts | 1 - apps/cli/tests/helpers/storage.ts | 1 - packages/stack/ARCHITECTURE.md | 14 +-- packages/stack/README.md | 16 +-- .../stack/src/HostProcess.integration.test.ts | 4 - .../stack/src/Network.integration.test.ts | 118 ++++++------------ packages/stack/src/Network.ts | 57 +++------ .../src/Orchestrator.integration.test.ts | 3 +- packages/stack/src/Orchestrator.ts | 32 +++-- packages/stack/src/Owner.integration.test.ts | 102 +++++++++++---- ...er.native-socket-sweep.integration.test.ts | 1 - packages/stack/src/Owner.ts | 13 +- packages/stack/src/Ports.integration.test.ts | 8 +- packages/stack/src/Ports.ts | 105 ++++++---------- ...ost.container-shutdown.integration.test.ts | 9 -- .../stack/src/StackHost.integration.test.ts | 12 -- packages/stack/src/StackNamespace.ts | 1 - packages/stack/src/Sweep.integration.test.ts | 1 - .../Supabase.native.integration.test.ts | 1 - .../src/composition/Supabase.unit.test.ts | 1 + packages/stack/src/effect.integration.test.ts | 2 - packages/stack/src/effect.ts | 1 - .../src/internal/dispatch.integration.test.ts | 1 - .../stack/src/lifetime.integration.test.ts | 3 - .../stack/src/namespace/Namespace.e2e.test.ts | 1 - .../namespace/Namespace.integration.test.ts | 1 - .../namespace/Namespace.windows.e2e.test.ts | 1 - packages/stack/src/namespace/Registry.ts | 8 -- .../stack/src/port-reservation.e2e.test.ts | 80 ------------ .../src/runtime/Container.integration.test.ts | 1 - .../src/services/Rest.integration.test.ts | 3 +- .../StorageGateway.integration.test.ts | 1 - .../stack/src/shutdown.integration.test.ts | 1 - packages/stack/tests/standalone-service.ts | 1 + 43 files changed, 226 insertions(+), 389 deletions(-) diff --git a/.oxlintrc.json b/.oxlintrc.json index 0d3ae82fba..36f4b46dbb 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -77,6 +77,7 @@ "packages/stack/src/namespace/**", "packages/stack/src/StackNamespace.ts", "packages/stack/src/Network.ts", + "packages/stack/src/Network.integration.test.ts", "packages/stack/src/Owner.ts", "packages/stack/src/Ports.ts", "packages/stack/src/Ports.integration.test.ts", @@ -256,6 +257,7 @@ // - DockerDatabaseStorage.ts: needs Publication. "files": [ "packages/stack/src/Network.ts", + "packages/stack/src/Network.integration.test.ts", "packages/stack/src/Owner.ts", "packages/stack/src/Ports.ts", "packages/stack/src/Ports.integration.test.ts", diff --git a/apps/cli/src/command-internal/db-config.integration.test.ts b/apps/cli/src/command-internal/db-config.integration.test.ts index fd3d0330ec..436fe889ef 100644 --- a/apps/cli/src/command-internal/db-config.integration.test.ts +++ b/apps/cli/src/command-internal/db-config.integration.test.ts @@ -505,7 +505,6 @@ describe("dbConfigResolver (db-url under the stack backend)", () => { instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }, host: undefined, }) diff --git a/apps/cli/src/commands/db/dump/dump.integration.test.ts b/apps/cli/src/commands/db/dump/dump.integration.test.ts index bc36e1d9e5..8aeff97c8e 100644 --- a/apps/cli/src/commands/db/dump/dump.integration.test.ts +++ b/apps/cli/src/commands/db/dump/dump.integration.test.ts @@ -168,7 +168,6 @@ const managedDumpStackApi = (runtime: "native" | "docker") => { instances: [], composition: { members: [{ id, activation: "eager" as const }], dependencies: [] }, lifetime: "detached" as const, - ports: [], }, host: undefined, }), diff --git a/apps/cli/src/commands/db/reset/reset.integration.test.ts b/apps/cli/src/commands/db/reset/reset.integration.test.ts index ee6ab2975d..d4695f30f2 100644 --- a/apps/cli/src/commands/db/reset/reset.integration.test.ts +++ b/apps/cli/src/commands/db/reset/reset.integration.test.ts @@ -826,7 +826,6 @@ function mockResetStackApi(opts: { })), lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }, host: undefined, }), diff --git a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts index 5be13eaaf0..27a8784256 100644 --- a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts @@ -180,7 +180,6 @@ function generateStackApi(workdir: string) { instances: [], composition, lifetime: "detached" as const, - ports: [], }, host: undefined, }), diff --git a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts index 5c572d651c..df21a46df3 100644 --- a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts @@ -182,7 +182,6 @@ function syncStackApi(workdir: string, port: number) { instances: [], composition, lifetime: "detached" as const, - ports: [], }, host: undefined, }), diff --git a/apps/cli/src/commands/db/start/start.integration.test.ts b/apps/cli/src/commands/db/start/start.integration.test.ts index 9c8e1be2c3..46c9c19d70 100644 --- a/apps/cli/src/commands/db/start/start.integration.test.ts +++ b/apps/cli/src/commands/db/start/start.integration.test.ts @@ -1753,7 +1753,6 @@ describe("db start stack backend", () => { instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }, host: undefined, }) diff --git a/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts index 31396b77a4..c998e74f68 100644 --- a/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts @@ -122,7 +122,6 @@ const definitionFor = (id: string, projectRoot: string, creation: ServiceCreatio runtime: "native", instances: [{ id: "database-id", creation }], composition: { members: [{ id: "database-id", activation: "eager" }], dependencies: [] }, - ports: [], }); interface TargetSpec { diff --git a/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts b/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts index 99f2c538db..793649418f 100644 --- a/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts @@ -238,7 +238,6 @@ const runStatus = (input: { })), composition: { members: input.members ?? [], dependencies: [] }, lifetime: "detached" as const, - ports: [], }; const api = Layer.succeed(StackApi, { create: () => Effect.die("create must not run"), diff --git a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts index dced26c90c..c7e2e04fe3 100644 --- a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts +++ b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts @@ -301,7 +301,6 @@ const fixture = ( instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }, host: undefined, }), diff --git a/apps/cli/tests/helpers/storage.ts b/apps/cli/tests/helpers/storage.ts index 03933f1982..04171be38d 100644 --- a/apps/cli/tests/helpers/storage.ts +++ b/apps/cli/tests/helpers/storage.ts @@ -274,7 +274,6 @@ export function buildStorageStackApi( ], composition: { members, dependencies: [] }, lifetime: "detached" as const, - ports: [], }; const findStackCalls: Array<{ readonly projectRoot: string }> = []; const layer = diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 06cd7f8585..24c4c81598 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -62,7 +62,7 @@ Organize by cohesive responsibilities. The package shape is: - `testing.ts`: disposable, composed session stacks for tests, with database checkpoints, in Promise and Effect forms. - `Defaults.ts`: shared local-development credentials, exported once as `./defaults`. -This is navigational guidance, not a required file scaffold. Split modules when a responsibility needs it; avoid one folder or interface per operation. Keep service definitions narrow, with graph edges and input wiring in composition. Do not introduce capabilities, projections, recovery journals, reservations, public sleep APIs, or extra lifecycle states to force this shape. +This is navigational guidance, not a required file scaffold. Split modules when a responsibility needs it; avoid one folder or interface per operation. Keep service definitions narrow, with graph edges and input wiring in composition. Do not introduce capabilities, projections, recovery journals, reservation mechanisms beyond the machine-wide per-OS-user port registry, public sleep APIs, or extra lifecycle states to force this shape. That registry is the one deliberate reservation and the only saved public-port assignment. Application services use Effect `Context.Service` and `Layer.effect`; consumers obtain their dependencies from the Effect context. Each owner receives an isolated orchestrator graph. Commands share the host lifetime alongside the owner. Individual executors, recipes, and process handles remain scoped resources because a stack owns multiple independently identified instances. @@ -312,7 +312,7 @@ Validate composition references, input/output types and cycles when registering Composition start first assigns or reuses and binds every configured public listener of its selected members, including lazy members, and registers their routes. One worker per member then waits for its prerequisites' outcomes: a lazy member is armed (also after a prerequisite failed), and an eager member is started and awaited for readiness, or reported as blocked when a prerequisite failed. Prerequisites outside the configured members run as eager standalone instances. It does not select unrelated standalone instances. Successful start returns member observations with their public endpoints; credentials reads render usable connection strings from these assignments without starting lazy backends. A failed launch or readiness check makes the operation return an error with per-instance outcomes; a running but unhealthy instance remains running, and completed steps are not rolled back. -After binding the shared API listener, default composition setup uses the endpoint renderer to populate ordinary configuration values: Studio receives `apiUrl` and `publicApiUrl`, Auth receives `externalApiUrl`, and Functions receives `apiUrl`. When the database has a configured SQL endpoint, Functions also receives its rendered `databaseUrl`; this saved value is refreshed when callers recompose the composition, and does not create a database readiness dependency for Function execution. These values are saved in each instance's configuration alongside the saved listener assignment; individual starts and normal host reopening consume that configuration. This is concrete default setup code, not declarative input wiring or a public listener-binding API. The endpoint renderer owns the host/runtime reachability rules; custom configurations supply plain URLs through the same inputs. URL values do not imply graph edges. +After binding the shared API listener, default composition setup uses the endpoint renderer to populate ordinary configuration values: Studio receives `apiUrl` and `publicApiUrl`, Auth receives `externalApiUrl`, and Functions receives `apiUrl`. When the database has a configured SQL endpoint, Functions also receives its rendered `databaseUrl`; this saved value is refreshed when callers recompose the composition, and does not create a database readiness dependency for Function execution. These values are saved in each instance's configuration; individual starts and normal host reopening consume that configuration. This is concrete default setup code, not declarative input wiring or a public listener-binding API. The endpoint renderer owns the host/runtime reachability rules; custom configurations supply plain URLs through the same inputs. URL values do not imply graph edges. Composition restart is two passes: stop the selected instances in reverse dependency order, then perform composition start. If the stop pass fails, report partial results without beginning the start pass. Cancelling composition withdraws its waits; admitted executions continue in the owner scope, and launched instances remain until an explicit stop or owner close. Explicit empty selection is a no-op; there is no generic rollback engine. @@ -357,7 +357,7 @@ flowchart TB Stopped -->|explicit start| Start["Orchestrator starts selected services
Reuses saved public ports"] ``` -Sleep retains the public listener needed to wake. Whole-stack stop closes listeners while preserving saved assignments; the next start rebinds the same ports. Destruction releases only the removed instance's owned assignments, not unrelated services' shared listeners or claims. +Sleep retains the public listener needed to wake. Whole-stack stop closes listeners while preserving the registry's reservations; the next start rebinds the same ports. Destroying an individual instance deletes only the registry rows it owns, not unrelated services' shared listeners or rows; a stack-wide destroy deletes its rows only after it fully succeeds, and a failed destroy keeps them. Composition validation permits lazy activation only for instances with a configured public wake endpoint. A route-less prerequisite, such as pg-meta without its own public endpoint, is eager; it is not implicitly armed through a dependent. This avoids a second wake-permission mechanism. Internal sleep is available only for lazy instances with a supported public wake route and an enabled idle policy. The default composition gives every lazy service one, Functions included; the database is eager. Functions inspector access remains possible while health is starting; ordinary application traffic waits for healthy. @@ -505,7 +505,7 @@ nothing left to update, and any resource claim it already recorded stays availab reconcile. Deletion, a move and an unmount are treated identically, because a stack whose root is unreachable cannot operate regardless of which of the three caused it. -**Unreachable engine.** An owner of a container stack fails startup with a `runtime-unavailable` reason when its first container sweep finds the engine CLI missing or its daemon not listening; permission, TLS, authentication and timeout failures are ordinary startup errors. `destroy` then proceeds without an owner: it takes the free lease, publishing a sweeper record like the orphan sweep, refuses when any stack data directory cannot be deleted by the current user, removes the host data and the registration with its port claims, and returns the shell commands that remove the stack's containers and engine-volume data once the engine runs. `stop` without a live owner takes the same lease and reclaims the registered stack's leftovers (the orphan-sweep path: containers, plus destruction of a session stack) before succeeding; it fails if another process holds the lease or the cleanup fails, and so needs the engine for a container stack. +**Unreachable engine.** An owner of a container stack fails startup with a `runtime-unavailable` reason when its first container sweep finds the engine CLI missing or its daemon not listening; permission, TLS, authentication and timeout failures are ordinary startup errors. `destroy` then proceeds without an owner: it takes the free lease, publishing a sweeper record like the orphan sweep, refuses when any stack data directory cannot be deleted by the current user, removes the host data, the registration and its port reservations, and returns the shell commands that remove the stack's containers and engine-volume data once the engine runs. `stop` without a live owner takes the same lease and reclaims the registered stack's leftovers (the orphan-sweep path: containers, plus destruction of a session stack) before succeeding; it fails if another process holds the lease or the cleanup fails, and so needs the engine for a container stack. During Serving, keep the owner alive independently of callers. Sleeping instances still need its public listeners. This is process lifetime management, not automatic service restart or continuous reconciliation. @@ -538,7 +538,7 @@ owner never serves again. A failed stop or destroy reports its error, the stack with uncertain state, and the owner exits. The next stop or destroy finds no owner and reclaims the stack under its lease. -**Successful whole-stack shutdown.** `stack.stop()` reports success only after admitted work settles, every owned live service and command workload has stopped (including native processes, descendants and containers labeled with this stack and data root), stack listeners close, the shutdown request is acknowledged, and the detached host's exit is confirmed. If workload cleanup cannot be confirmed, stop fails and the host exits without resuming service; the stack is not reported stopped and remains registered for the next stop or destroy to reclaim. If cleanup succeeds but host exit cannot be confirmed, stop also fails, though the host may already have exited. A delivered SIGTERM or SIGINT follows this cleanup path. Stop preserves stack definitions, service data, caches and saved port assignments. `destroy` follows the same live-workload cleanup, then removes only proven-owned persistent data. +**Successful whole-stack shutdown.** `stack.stop()` reports success only after admitted work settles, every owned live service and command workload has stopped (including native processes, descendants and containers labeled with this stack and data root), stack listeners close, the shutdown request is acknowledged, and the detached host's exit is confirmed. If workload cleanup cannot be confirmed, stop fails and the host exits without resuming service; the stack is not reported stopped and remains registered for the next stop or destroy to reclaim. If cleanup succeeds but host exit cannot be confirmed, stop also fails, though the host may already have exited. A delivered SIGTERM or SIGINT follows this cleanup path. Stop preserves stack definitions, service data, caches and port reservations. `destroy` follows the same live-workload cleanup, then removes only proven-owned persistent data. The client captures the live owner PID from the validated identity endpoint or readiness handshake, completes the shutdown request, then performs bounded process-existence checks. An absent PID confirms exit; a permission-denied probe remains inconclusive until the deadline. Failure to confirm exit is a `shutdown-exit` error carrying the PID in its message, even when workload cleanup has already succeeded. This does not require persisted PID records or forceful termination. Caller cancellation ends its wait without cancelling admitted owner cleanup. @@ -592,7 +592,7 @@ A composition returns the observations of its members. Remove the separate capab ### Persist only what normal stop/start needs -Retain stack/instance identity, service configuration and selected versions, composition membership/wiring, credentials, public port assignments and the service-specific information needed to reopen the data. Derive stack and instance data locations from the state root and saved identities rather than persisting duplicate paths. Keep the data itself across stop. Shared port claims remain necessary for stickiness across parallel stacks. The active host's exclusive lease prevents competing owners; it is not a crash-recovery subsystem. +Retain stack/instance identity, service configuration and selected versions, composition membership/wiring, credentials and the service-specific information needed to reopen the data. Derive stack and instance data locations from the state root and saved identities rather than persisting duplicate paths. Keep the data itself across stop. Public port assignments are not part of the stack document: the machine-wide port registry (`ports.sqlite`) holds the only saved assignment and keeps ports sticky across parallel stacks. The active host's exclusive lease prevents competing owners; it is not a crash-recovery subsystem. Do not persist health, runtime lifecycle projections, transition progress, command queues, operation results or recovery checkpoints. A disconnected caller can reconnect to the same living host; resuming after host death is not part of the contract. Incomplete operations and leftover resources after a crash may require manual cleanup, without automatic deletion of valuable data. @@ -626,7 +626,7 @@ A shared listener is a networking resource, not a capability or service group. C A lazy database already has a bound SQL proxy listener and a concrete connection URL; lazy REST already has its path on the bound shared API listener. Their backend processes may still be absent. The first request launches the needed backends, waits for readiness and forwards through the already published endpoint. Backend address allocation can wait until that launch; public port assignment cannot. If a required listener cannot bind, startup returns an error with partial outcomes rather than reporting successful startup with unusable URLs. -A host-wide allocation registry coordinates claims across parallel stacks, including stopped instances and stacks. Before first listener activation, obtain an assignment and bind it. Automatic allocation excludes managed claims and tries to bind an available candidate. Persist the assignment before advertising it and retain that socket; do not probe then release it. Later activations bind the saved port. Fixed requests obey the same ownership and binding checks. +A host-wide allocation registry coordinates claims across parallel stacks, including stopped instances and stacks. Before first listener activation, obtain an assignment and bind it. Automatic allocation excludes managed claims and tries to bind an available candidate. Reserve the assignment in the registry before advertising it and retain that socket; do not probe then release it. Later activations bind the saved port. Fixed requests obey the same ownership and binding checks. ```mermaid flowchart LR diff --git a/packages/stack/README.md b/packages/stack/README.md index 151c0927c1..f7cf7ebd1e 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -57,7 +57,7 @@ await stack.close(); // disconnects this client `start` and `ready` are separate operations. `restart({ config })` replaces recipe configuration while retaining the instance identity and endpoint intentions. A health failure leaves a launched process running and observable; it does not prevent `stop`. Database snapshots require a stopped instance with wake disabled. `saveSnapshot(key)` publishes complete data to managed backend storage and replaces the previous entry for that key; `restoreSnapshot(key)` returns `false` on a miss and `true` after restoring a compatible entry. By default snapshots live in the shared cache, whose retention may evict older keys, and survive destruction of the source stack. `{ scope: "instance" }` keeps a snapshot with the instance instead: it is never evicted, restores only into that instance, survives `resetData`, and is removed when the instance is destroyed. Other service handles have no snapshot methods. -Creating a service records its definition. Configured public ports are bound during startup and retained across normal stop/start and owner reopening. An occupied saved port reports a conflict instead of moving. Automatic ports avoid numbers saved by any stack under the same `stateRoot`; a fixed port is decided by binding it, so another stack's saved port blocks only while something listens on it; that conflict names the stack that saved the port. Omitted public endpoints are not exposed. +Creating a service records its definition. Configured public ports are bound during startup and retained across normal stop/start and owner reopening. An occupied saved port reports a conflict instead of moving. Automatic ports avoid every port reserved in the per-user port registry (see Public ports); a fixed port another stack holds is a conflict that names that stack. Omitted public endpoints are not exposed. Native public listeners bind to loopback. Docker public proxies bind all interfaces so services inside the container network can reach them; those listeners are reachable from the LAN according to the host firewall. @@ -65,7 +65,7 @@ Functions use a package-provided, self-contained Edge Runtime main service unles On Linux, native Functions project files must be outside `/tmp`: Edge Runtime uses a private filesystem at that path. Docker mounts project files at a separate runtime path. -`open({ id, stateRoot, cacheRoot })` reconnects to a saved stack. The package stores the stack document at `//state.json` and service data at `//data/`. `discover({ stateRoot })` lists saved definitions and port assignments separately from live-owner availability. It skips each entry that cannot be read or decoded and reports it to `onInvalidState(id, error)`; only a failure to read `stateRoot` itself fails discovery. Port allocation skips the same entries. Offline definitions are not live lifecycle observations. +`open({ id, stateRoot, cacheRoot })` reconnects to a saved stack. The package stores the stack document at `//state.json` and service data at `//data/`. `discover({ stateRoot })` lists saved definitions separately from live-owner availability. It skips each entry that cannot be read or decoded and reports it to `onInvalidState(id, error)`; only a failure to read `stateRoot` itself fails discovery. Port allocation skips the same entries. Offline definitions are not live lifecycle observations. `find({ stateRoot, projectRoot, name })` derives the stack ID with the same identity rules as `create` and reads only that stack; `find({ stateRoot, id })` reads a known ID. It returns the saved definition with the live owner's endpoint, if any, or nothing when no such stack is saved. Unlike `discover`, an unreadable state document fails the call instead of being skipped. The Effect entrypoint's `StackId` schema validates an ID before lookup. @@ -73,11 +73,11 @@ Pass `startOwner: true` to `open` when live status and other owner-backed operat ### Lifetimes -`create` defaults to `lifetime: "detached"`: the stack and its owner outlive the creating client. `create({ ..., lifetime: "session" })` starts the owner immediately and ties the stack to the creating client. Closing that client, or its process exiting for any reason, destroys the stack: instances stop, data and registrations are removed, and port claims are released. Other clients may attach to a running session stack but cannot start its owner. A session stack whose owner has stopped is disposable: the next owner start in the state root removes it, and creating a stack with the same identity replaces it. +`create` defaults to `lifetime: "detached"`: the stack and its owner outlive the creating client. `create({ ..., lifetime: "session" })` starts the owner immediately and ties the stack to the creating client. Closing that client, or its process exiting for any reason, destroys the stack: instances stop, data and registrations are removed, and port reservations are released. Other clients may attach to a running session stack but cannot start its owner. A session stack whose owner has stopped is disposable: the next owner start in the state root removes it, and creating a stack with the same identity replaces it. `create({ ..., startOwner: true })` starts a detached stack's owner immediately and lets it register the stack under its lease, as a session stack always does. If the owner fails to start or the launch is interrupted before it reports ready, the owner removes that registration and `create` fails with the launch error, so a failed launch leaves no stack behind. -`destroy` normally returns `{ runtimeCleanup: "complete" }`. When no owner is running and a new owner cannot start because the stack's container engine reports that its daemon cannot be reached, `destroy` confirms that no owner holds the stack's lease and leaves the registration, port claims and host data untouched, returning `{ runtimeCleanup: "skipped", engine }`. When an owner does run but a labelled container cannot be removed (for example one on a different daemon) or an instance's data cannot be removed, `destroy` likewise keeps the stack registered and fails, listing what remains. Either way, retry `destroy` once the engine is reachable; there is no `cleanupCommands` field. +`destroy` normally returns `{ runtimeCleanup: "complete" }`. When no owner is running and a new owner cannot start because the stack's container engine reports that its daemon cannot be reached, `destroy` confirms that no owner holds the stack's lease and leaves the registration, port reservations and host data untouched, returning `{ runtimeCleanup: "skipped", engine }`. When an owner does run but a labelled container cannot be removed (for example one on a different daemon) or an instance's data cannot be removed, `destroy` likewise keeps the stack registered and fails, listing what remains. Either way, retry `destroy` once the engine is reachable; there is no `cleanupCommands` field. The stack owns database, Functions bootstrap, and command-job directories below its data directory. Storage uploads remain at the caller-supplied Storage `filePath` and are preserved when the stack is destroyed; the caller owns that directory. Host metadata remains under `stateRoot`; native database data uses host files. Docker database data normally uses a managed volume, while existing host data is retained through the host-backed fallback. A host marker records the selected Docker storage and detects a missing or mismatched volume; deleting that volume loses the associated database data. Native snapshot entries live below `cacheRoot`. Docker snapshots share the managed data volume in a separate namespace derived from `cacheRoot`, so they survive source destruction and can use filesystem cloning. A Docker cache hit requires the same daemon, `stateRoot`, and `cacheRoot`. There is no portable tar snapshot API. @@ -97,15 +97,15 @@ Every public port is reserved, before any listener binds it, in one SQLite regis `/.supabase/ports.sqlite`. The home is resolved once per process straight from the OS user database (`getent passwd ` on Linux, `dscacheutil -q user -a uid ` on macOS), never `$HOME` or `SUPABASE_HOME`, and with no override; a process cannot steer the registry's location by -setting its own environment. The registry is the only authority on which stack owns a port across +setting its own environment. The registry is the only saved port assignment (the stack's `state.json` holds none) and the only authority on which stack owns a port across every state root on the machine, so a stopped stack keeps its ports across other stacks' starts, and a restart reuses the same ports or fails with a `StackError` whose `conflict` field names the port, the endpoint, and either the live holder (`{ stackId, stateRoot }`) or `"foreign"` for a process outside the registry. A fixed or previously saved port is never silently reassigned; only automatic allocation tries another candidate. A reservation is released when -`destroy` confirms the stack has nothing left to clean up, or when deleting an individual service -releases the endpoints only it owned; stopping a stack or killing its owner leaves reservations in -place. A reservation whose owning stack's `state.json` is confirmed gone (`ENOENT`, for example +deleting an individual service releases the endpoints only it owned, or when a stack-wide `destroy` +has fully succeeded; a failed `destroy` keeps every reservation, and stopping a stack or killing its +owner leaves them in place. A reservation whose owning stack's `state.json` is confirmed gone (`ENOENT`, for example after deleting its state root) is reclaimed lazily by the next stack that needs its port. Known limitations: on macOS, BSD, and Windows, where the kernel allows overlapping binds, a loopback diff --git a/packages/stack/src/HostProcess.integration.test.ts b/packages/stack/src/HostProcess.integration.test.ts index b5ef0c5093..6d777c0bfd 100644 --- a/packages/stack/src/HostProcess.integration.test.ts +++ b/packages/stack/src/HostProcess.integration.test.ts @@ -47,7 +47,6 @@ const savedStack = (root: string, stackName: string): StackNamespace.SavedStack instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); /** A loopback listener that accepts connections and never answers, counting each one. */ @@ -265,7 +264,6 @@ it.live("starts exactly one owner for concurrent launchers and attaches the othe instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const entrypoint = fileURLToPath( new URL("../tests/host-process-fixture.ts", import.meta.url), @@ -419,7 +417,6 @@ it.live( instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const entrypoint = fileURLToPath( new URL("../tests/host-process-fixture.ts", import.meta.url), @@ -454,7 +451,6 @@ it.live("terminates a detached child when readiness is interrupted", () => instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const entrypoint = fileURLToPath( new URL("../tests/host-process-fixture.ts", import.meta.url), diff --git a/packages/stack/src/Network.integration.test.ts b/packages/stack/src/Network.integration.test.ts index edf144b9de..57f96dfb42 100644 --- a/packages/stack/src/Network.integration.test.ts +++ b/packages/stack/src/Network.integration.test.ts @@ -5,6 +5,7 @@ import * as Net from "node:net"; // oxlint-disable-line effecttsgo/node-builtin- import { createServer } from "node:http"; // oxlint-disable-line effecttsgo/node-builtin-import -- real socket fixture. import { HttpClient } from "effect/unstable/http"; import * as Network from "./Network.ts"; +import * as PortReservations from "./namespace/PortReservations.ts"; import { DOCKER_HOST_ALIAS } from "./runtime/Container.ts"; import * as StackNamespace from "./StackNamespace.ts"; @@ -23,16 +24,23 @@ const makeTestNetwork = (options: { ), ).pipe(Effect.map((context) => Context.get(context, Network.Service))); -const stack = (id: string, port: number | "auto") => ({ +const stack = (id: string) => ({ id, identity: { projectRoot: "/tmp", branchContext: "test", stackName: id }, runtime: "native" as const, lifetime: "detached" as const, instances: [], composition: { members: [], dependencies: [] }, - ports: port === "auto" ? [] : [{ key: "api", host: "127.0.0.1", port }], }); +/** The port the machine-wide registry holds for this stack's endpoint key, if any. */ +const reservedPort = (root: string, stackId: string, key: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const registry = yield* PortReservations.Service; + return yield* registry.find(yield* fs.realPath(root), stackId, key); + }).pipe(Effect.provide(PortReservations.layer)); + class FixtureError extends Data.TaggedError("FixtureError")<{ readonly message: string }> {} const backend = Effect.acquireRelease( @@ -101,7 +109,7 @@ it.live("retains dedicated assignments across network reopen", () => const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-retain-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const enabled = yield* Ref.make(true); const first = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); @@ -112,8 +120,7 @@ it.live("retains dedicated assignments across network reopen", () => yield* firstNamespace.bind; const firstAddress = yield* firstNamespace.address("api", "host"); expect(yield* request(firstAddress.host, firstAddress.port, "/one")).toBe("backend:/one"); - const saved = yield* state.read("stack"); - expect(saved?.ports).toHaveLength(1); + expect(yield* reservedPort(root, "stack", "one:api")).toBe(firstAddress.port); yield* Ref.set(enabled, false); yield* firstNamespace.close; yield* firstNamespace.bind; @@ -134,7 +141,9 @@ it.live("retains dedicated assignments across network reopen", () => "backend:/reopen", ); yield* secondNamespace.release; - expect((yield* state.read("stack"))?.ports).toHaveLength(0); + expect(yield* reservedPort(root, "stack", "one:api")).toBe(firstAddress.port); + yield* secondNamespace.releasePorts; + expect(yield* reservedPort(root, "stack", "one:api")).toBeUndefined(); expect(yield* fs.exists(path.join(root, "stack", "state.json"))).toBe(true); }), ).pipe(Effect.provide(NodeServices.layer)), @@ -146,7 +155,7 @@ it.live("keeps shared routes independent and retains the shared claim", () => const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-shared-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const firstEnabled = yield* Ref.make(true); const secondEnabled = yield* Ref.make(true); @@ -186,7 +195,7 @@ it.live("keeps shared routes independent and retains the shared claim", () => expect(yield* request(address.host, address.port, "/auth/v1")).toBe("backend:/auth/v1"); yield* Ref.set(secondEnabled, false); yield* second.close; - expect((yield* state.read("stack"))?.ports).toHaveLength(1); + expect(yield* reservedPort(root, "stack", "api")).toBe(address.port); yield* first.bind; expect((yield* first.address("api", "host")).port).toBe(address.port); expect(yield* request(address.host, address.port, "/rest/reopen")).toBe( @@ -194,20 +203,21 @@ it.live("keeps shared routes independent and retains the shared claim", () => ); yield* first.close; yield* second.release; - expect((yield* state.read("stack"))?.ports).toHaveLength(1); + yield* second.releasePorts; + expect(yield* reservedPort(root, "stack", "api")).toBe(address.port); expect(yield* fs.exists(root)).toBe(true); }), ).pipe(Effect.provide(NodeServices.layer)), ); -it.live("reports a dedicated port conflict without rewriting saved ownership", () => +it.live("reports a dedicated port conflict without taking over the holder's reservation", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-conflict-" }); const state = yield* makeTestState(root); - yield* state.save(stack("first", "auto")); - yield* state.save(stack("second", 20001)); + yield* state.save(stack("first")); + yield* state.save(stack("second")); const target = yield* backend; const firstEnabled = yield* Ref.make(true); const firstNetwork = yield* makeTestNetwork({ stackId: "first", runtime: "native", state }); @@ -231,60 +241,14 @@ it.live("reports a dedicated port conflict without rewriting saved ownership", ( }); const failure = yield* second.bind.pipe(Effect.flip); expect(failure.operation).toBe("bind"); - expect((yield* state.read("second"))?.ports).toEqual([ - { key: "api", host: "127.0.0.1", port: 20001 }, - ]); + expect(yield* reservedPort(root, "first", "db:sql")).toBe(firstPort); + expect(yield* reservedPort(root, "second", "db:sql")).toBeUndefined(); yield* Ref.set(firstEnabled, false); yield* first.release; + yield* first.releasePorts; yield* Ref.set(secondEnabled, false); yield* second.release; - }), - ).pipe(Effect.provide(NodeServices.layer)), -); - -it.live("can bind a shared listener after saving its first assignment failed", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-save-" }); - const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); - const failing = yield* Ref.make(true); - const network = yield* makeTestNetwork({ - stackId: "stack", - runtime: "native", - state: { - ...state, - save: (value) => - Ref.get(failing).pipe( - Effect.flatMap((fail) => - fail - ? Effect.fail( - new StackNamespace.NamespaceError({ - operation: "save", - message: "write failed", - }), - ) - : state.save(value), - ), - ), - }, - }); - const target = yield* backend; - const namespace = yield* network.register({ - id: "rest", - endpoints: { - api: { ...endpoint(target, Effect.succeed(false)), shared: [{ prefix: "/rest" }] }, - }, - }); - yield* namespace.bind.pipe(Effect.flip); - yield* Ref.set(failing, false); - yield* namespace.bind; - const address = yield* namespace.address("api", "host"); - expect(yield* request(address.host, address.port, "/rest")).toBe("backend:/rest"); - yield* namespace.release; - yield* network.release; - expect((yield* state.read("stack"))?.ports).toHaveLength(0); + yield* second.releasePorts; }), ).pipe(Effect.provide(NodeServices.layer)), ); @@ -295,7 +259,7 @@ it.live("releases dedicated HTTP activity after the response while keep-alive st const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-http-activity-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const released = yield* Deferred.make(); const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); @@ -350,7 +314,7 @@ it.live( const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-join-fixed-port-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); const studio = yield* network.register({ @@ -360,8 +324,7 @@ it.live( yield* studio.bind; // The join route is queued, not installed: it never claims or asserts the shared "api" // port, so no shared listener or claim exists yet, whatever port a later claimant picks. - const beforeClaimant = yield* state.read("stack"); - expect(beforeClaimant?.ports.some((claim) => claim.key === "api")).toBe(false); + expect(yield* reservedPort(root, "stack", "api")).toBeUndefined(); const rest = yield* network.register(claimant("rest", target, Effect.succeed(true))); yield* rest.bind; const address = yield* rest.address("api", "host"); @@ -379,7 +342,7 @@ it.live("drops a queued join route when its namespace closes before any claimant const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-join-cancel-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); const studio = yield* network.register({ @@ -407,7 +370,7 @@ it.live("stays idempotent across repeated binds of the joining namespace", () => const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-join-idempotent-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); const rest = yield* network.register(claimant("rest", target, Effect.succeed(true))); @@ -433,7 +396,7 @@ it.live("keeps a namespace's own shared route when its join endpoint also binds" prefix: "network-join-shares-namespace-", }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); const combo = yield* network.register({ @@ -460,7 +423,7 @@ it.live("re-adds a joined route after its namespace closes and rebinds", () => const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-join-rebind-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const enabled = yield* Ref.make(true); const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); @@ -493,7 +456,7 @@ it.live("never restores a joined route once its namespace is released", () => const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-join-release-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const enabled = yield* Ref.make(true); const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); @@ -527,7 +490,7 @@ it.live( const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-join-last-close-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const restEnabled = yield* Ref.make(true); const studioEnabled = yield* Ref.make(true); @@ -582,7 +545,7 @@ it.live("wakes a sleeping backend when a request reaches its joined route", () = const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-join-wake-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const woken = yield* Deferred.make(); const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); @@ -614,7 +577,7 @@ it.live("addresses docker runtime endpoints through the stack host alias", () => const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-docker-alias-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const network = yield* makeTestNetwork({ stackId: "stack", runtime: "docker", state }); const namespace = yield* network.register({ @@ -627,6 +590,7 @@ it.live("addresses docker runtime endpoints through the stack host alias", () => expect(host.host).toBe("127.0.0.1"); expect(runtime).toEqual({ ...host, host: DOCKER_HOST_ALIAS }); yield* namespace.release; + yield* namespace.releasePorts; }), ).pipe(Effect.provide(NodeServices.layer)), ); @@ -637,7 +601,7 @@ it.live("drains past an idle HTTP keep-alive connection without ever reaching th const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-drain-idle-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); const service = yield* network.register({ @@ -684,7 +648,7 @@ it.live("drains past an idle TCP connection without ever reaching the deadline", const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-drain-tcp-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); const service = yield* network.register({ @@ -728,7 +692,7 @@ it.live( const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-drain-inflight-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const services = yield* Effect.context(); const requested = yield* Deferred.make(); const release = yield* Deferred.make(); diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index a3979cda11..380224480a 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -69,7 +69,10 @@ export interface NetworkNamespace { readonly bindings: Effect.Effect>; readonly bind: Effect.Effect, NetworkError>; readonly close: Effect.Effect; + /** Closes the endpoints for good once the instance is stopped; saved port assignments stay. */ readonly release: Effect.Effect; + /** Deletes the saved port assignments of this instance's dedicated endpoints. */ + readonly releasePorts: Effect.Effect; readonly address: ( name: string, from: "host" | "runtime", @@ -77,16 +80,6 @@ export interface NetworkNamespace { } export interface Interface { - readonly release: Effect.Effect; - /** - * Marks a stack-wide destroy in progress: every per-instance `NetworkNamespace.release` closes - * its listeners as usual but skips deleting its own reservation rows, so a destroy that later - * turns out uncertain (remaining resource claims) still finds them. Call {@link releaseStack} once - * destroy is confirmed to leave nothing behind, or {@link cancelDestroy} to resume immediate - * per-service deletion (the shape an individual `destroyService` still relies on). - */ - readonly beginDestroy: Effect.Effect; - readonly cancelDestroy: Effect.Effect; /** Releases every reservation this stack holds, dedicated and shared alike, in one step. */ readonly releaseStack: Effect.Effect; readonly register: (options: { @@ -125,7 +118,6 @@ const makeNetwork = (options: { const probe = probeVacant(options.platform ?? process.platform); const owner = yield* Scope.Scope; const gate = yield* Semaphore.make(1); - const deferReservations = yield* Ref.make(false); // Set once under `gate` by `drain`, before it snapshots `listeners`, and never cleared; checked // under the same gate by `bind`, so a bind that hasn't yet acquired the gate when drain starts // either completes before drain's snapshot (and so is captured by it) or observes this and @@ -364,37 +356,35 @@ const makeNetwork = (options: { "Stop the instance before releasing its endpoints", ); yield* Ref.set(closed, true); - // A stack-wide destroy defers every row deletion to its own success boundary (see - // `beginDestroy`), so an uncertain destroy (remaining resource claims) still finds - // this instance's reservations; an individual `destroyService` still deletes them now. - if (yield* Ref.get(deferReservations)) return; - for (const name of Object.keys(endpoints)) { - const endpoint = endpoints[name]; - if (endpoint?.shared === undefined) - yield* ports - .release(options.stackId, `${id}:${name}`) - .pipe(Effect.mapError((cause) => errorFor("release", cause))); - } }), ), ), ); + const releasePorts = Effect.fn("Network.releasePorts")(() => + Effect.forEach( + Object.entries(endpoints).filter(([, endpoint]) => endpoint.shared === undefined), + ([name]) => + ports + .release(options.stackId, `${id}:${name}`) + .pipe(Effect.mapError((cause) => errorFor("release", cause))), + { discard: true }, + ), + ); const address = Effect.fn("Network.address")((name: string, from: "host" | "runtime") => Effect.gen(function* () { const endpoint = endpoints[name]; if (endpoint === undefined) return yield* errorFor("address", `Unknown endpoint ${name}`); - const saved = yield* options.state - .read(options.stackId) - .pipe(Effect.mapError((cause) => errorFor("address", cause))); const key = endpoint.shared === undefined ? `${id}:${name}` : "api"; - const claim = saved?.ports.find((value) => value.key === key); - if (claim === undefined) + const port = yield* ports + .assigned(options.stackId, key) + .pipe(Effect.mapError((cause) => errorFor("address", cause))); + if (port === undefined) return yield* errorFor("address", `Endpoint ${name} is not assigned`); return { name, protocol: endpoint.protocol, host: from === "host" ? hostAddress : runtimeAddress, - port: claim.port, + port, }; }), ); @@ -402,18 +392,12 @@ const makeNetwork = (options: { bind: bind(), close: close(), release: release(), + releasePorts: releasePorts(), address, bindings: Ref.get(bound).pipe(Effect.map((values) => [...values.values()])), } satisfies NetworkNamespace; }); - const release = Effect.fn("Network.releaseNamespace")(() => - gate.withPermits(1)( - ports - .release(options.stackId, "api") - .pipe(Effect.mapError((cause) => errorFor("release", cause))), - ), - ); const releaseStack = Effect.fn("Network.releaseStack")(() => ports .releaseStack(options.stackId) @@ -452,9 +436,6 @@ const makeNetwork = (options: { }); return { register, - release: release(), - beginDestroy: Ref.set(deferReservations, true), - cancelDestroy: Ref.set(deferReservations, false), releaseStack: releaseStack(), drain: drain(), } satisfies Interface; diff --git a/packages/stack/src/Orchestrator.integration.test.ts b/packages/stack/src/Orchestrator.integration.test.ts index 831c85e110..7d68eecdf7 100644 --- a/packages/stack/src/Orchestrator.integration.test.ts +++ b/packages/stack/src/Orchestrator.integration.test.ts @@ -105,6 +105,7 @@ const makeInstance = ( bind: options.bind ?? Ref.set(bound, true), confirmRemoved: Effect.void, release: Effect.void, + releasePorts: Effect.void, close: orchestrator.status(id).pipe( Effect.flatMap((state) => state.lifecycle === "stopped" && !state.wakeEnabled ? Ref.set(bound, false) : Effect.void, @@ -1156,7 +1157,6 @@ describe("idle races through a real listener", () => { lifetime: "detached", instances: [], composition: { members: [], dependencies: [] }, - ports: [], }); const network = yield* Layer.build( Network.layer({ stackId, runtime: "native" }).pipe( @@ -1254,6 +1254,7 @@ describe("idle races through a real listener", () => { close: Effect.void, confirmRemoved: Effect.void, release: Effect.void, + releasePorts: Effect.void, hasEndpoint: true, inputs: [], outputs: {}, diff --git a/packages/stack/src/Orchestrator.ts b/packages/stack/src/Orchestrator.ts index 444e2d8f6b..c95939a7ac 100644 --- a/packages/stack/src/Orchestrator.ts +++ b/packages/stack/src/Orchestrator.ts @@ -81,12 +81,13 @@ export interface RegisteredInstance { */ readonly confirmRemoved: Effect.Effect; /** - * Releases this instance's network reservations once its data is confirmed removed: destroy's - * own step, run right after `core.removeData` succeeds. Abandonment never calls it — a stale - * reservation stays for `Ports.ts`'s lazy reclamation once the holder's registration is - * confirmed gone, the abandonment trigger, so abandonment has nothing to release itself. + * Closes this instance's network endpoints for good once its data is confirmed removed. Never + * deletes saved port assignments; abandonment never calls it, and a stale reservation stays for + * `Ports.ts`'s lazy reclamation once the holder's registration is confirmed gone. */ readonly release: Effect.Effect; + /** Deletes this instance's saved port assignments; an individual destroy runs it, a stack-wide destroy leaves them to `Network.releaseStack`. */ + readonly releasePorts: Effect.Effect; readonly hasEndpoint: boolean; readonly inputs: ReadonlyArray; readonly outputs: Readonly>>; @@ -956,8 +957,16 @@ export const make = Effect.fn("Orchestrator.make")(function* dependency.from === id); @@ -981,7 +990,12 @@ export const make = Effect.fn("Orchestrator.make")(function* destroyInstance(id, "delete"); + const settleOutcomes = Effect.fn("Orchestrator.settleOutcomes")(function* ( operation: "start" | "stop" | "destroy", ids: ReadonlyArray, @@ -1114,7 +1130,7 @@ export const make = Effect.fn("Orchestrator.make")(function* - destroy(id).pipe( + destroyInstance(id, "retain").pipe( Effect.exit, Effect.map((result) => ({ id, result })), ), diff --git a/packages/stack/src/Owner.integration.test.ts b/packages/stack/src/Owner.integration.test.ts index ddb48eed3b..daab355b60 100644 --- a/packages/stack/src/Owner.integration.test.ts +++ b/packages/stack/src/Owner.integration.test.ts @@ -33,7 +33,6 @@ const initial = (id: string): SavedStack => ({ instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const initialDocker = (id: string): SavedStack => ({ ...initial(id), runtime: "docker" }); @@ -777,37 +776,92 @@ it.effect("lets a retry choose other credentials after the first database creati ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); -it.effect("rejects a duplicate instance without releasing the existing instance's claims", () => +it.effect( + "rejects a duplicate instance without releasing the existing instance's reservations", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-duplicate-" }); + const state = yield* stateFor(`${root}/state`); + const creation = yield* Schema.decodeEffect(ServiceCreation)({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + const instance = { id: "mail", creation }; + const stack: SavedStack = { + ...initial(`owner-duplicate-${randomUUID().slice(0, 8)}`), + instances: [instance], + }; + yield* state.save(stack); + const realStateRoot = yield* fs.realPath(`${root}/state`); + const portReservations = yield* PortReservations.Service; + expect(yield* portReservations.reserve(realStateRoot, stack.id, "mail:http", 54_321)).toBe( + undefined, + ); + yield* Effect.addFinalizer(() => + portReservations.release(realStateRoot, stack.id, "mail:http").pipe(Effect.ignore), + ); + + const failure = yield* ownerFor({ + saved: { ...stack, instances: [instance, instance] }, + state, + root: `${root}/data`, + cacheRoot, + }).pipe(Effect.flip); + + expect(failure.message).toBe("Duplicate instance mail"); + expect(yield* portReservations.find(realStateRoot, stack.id, "mail:http")).toBe(54_321); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), +); + +it.live("destroying one service deletes its port reservation", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-duplicate-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-service-destroy-" }); + const stack = initial(`owner-service-destroy-${randomUUID().slice(0, 8)}`); const state = yield* stateFor(`${root}/state`); - const creation = yield* Schema.decodeEffect(ServiceCreation)({ - service: "mail", - config: {}, - endpoints: { http: { port: "auto" } }, - }); - const instance = { id: "mail", creation }; - const claim = { key: "mail:http", host: "127.0.0.1", port: 54_321 }; - const stack: SavedStack = { - ...initial("owner-duplicate"), - instances: [instance], - ports: [claim], - }; yield* state.save(stack); + const owner = yield* ownerFor({ saved: stack, state, root: `${root}/data`, cacheRoot }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + // Composing reserves every member's public port up front without starting any of them. + const definitions = yield* owner.rpc.supabaseComposition({ + services: [{ service: "mail", config: {}, endpoints: { http: { port: "auto" } } }], + }); + const mail = definitions.find((entry) => entry.creation.service === "mail"); + if (mail === undefined) return yield* Effect.die("mail missing from the composition"); + const realStateRoot = yield* fs.realPath(`${root}/state`); + const portReservations = yield* PortReservations.Service; + expect( + yield* portReservations.find(realStateRoot, stack.id, `${mail.id}:http`), + ).toBeDefined(); - const failure = yield* ownerFor({ - saved: { ...stack, instances: [instance, instance] }, - state, - root: `${root}/data`, - cacheRoot, - }).pipe(Effect.flip); + yield* owner.rpc.destroyService({ id: mail.id }); - expect(failure.message).toBe("Duplicate instance mail"); - expect((yield* state.read(stack.id))?.ports).toEqual([claim]); + expect( + yield* portReservations.find(realStateRoot, stack.id, `${mail.id}:http`), + ).toBeUndefined(); }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), ); it.effect("refuses to generate credentials for a stack whose saved instances consume them", () => diff --git a/packages/stack/src/Owner.native-socket-sweep.integration.test.ts b/packages/stack/src/Owner.native-socket-sweep.integration.test.ts index 22bbc97c49..0cd0035c5f 100644 --- a/packages/stack/src/Owner.native-socket-sweep.integration.test.ts +++ b/packages/stack/src/Owner.native-socket-sweep.integration.test.ts @@ -52,7 +52,6 @@ const arrange = Effect.fn("arrange")(function* ( instances: [{ id: "database", creation: yield* databaseCreation }], lifetime, composition: { members: [], dependencies: [] }, - ports: [], }; yield* state.save(saved); const runtimeRootBase = yield* fs.makeTempDirectoryScoped({ prefix: "stack-socket-sweep-base-" }); diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index 68942ca9db..e175adb768 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -501,6 +501,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { namespace, confirmRemoved: removeInstanceRegistration(id).pipe(Effect.mapError(serviceError("state"))), release: namespace.release.pipe(Effect.mapError(serviceError("release"))), + releasePorts: namespace.releasePorts.pipe(Effect.mapError(serviceError("release"))), launch: (generation, inputs, candidate) => configFor(inputs, candidate).pipe( Effect.flatMap((config) => core.launch(generation, config)), @@ -794,20 +795,14 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { definitionGate.withPermits(1), Effect.withSpan("Owner.stopNamespace"), ), - destroy: network.beginDestroy.pipe( - // Deferred for the whole of this destroy: every instance's own teardown below closes - // its listeners as usual, but leaves its reservation rows for `releaseStack` to drop - // together, only once destroy is confirmed to leave nothing behind. `ensuring` below - // resumes immediate per-service deletion again on every exit, success, failure, or - // interruption alike, so a service destruction or sweep failure can never leave - // deferral stuck on for a stack that is still otherwise live. - Effect.andThen(network.drain), + // Every instance's teardown closes its listeners but retains its saved port assignments; + // `releaseStack` drops them together, only once destroy has fully succeeded. + destroy: network.drain.pipe( Effect.andThen(orchestrator.destroyNamespace), Effect.andThen(sweep), Effect.andThen(removeContainerEnvRoot), Effect.andThen(network.releaseStack), Effect.andThen(options.state.remove(stackId)), - Effect.ensuring(network.cancelDestroy), definitionGate.withPermits(1), Effect.withSpan("Owner.destroyNamespace"), ), diff --git a/packages/stack/src/Ports.integration.test.ts b/packages/stack/src/Ports.integration.test.ts index 8f2b6a82cf..3d40882249 100644 --- a/packages/stack/src/Ports.integration.test.ts +++ b/packages/stack/src/Ports.integration.test.ts @@ -34,7 +34,7 @@ import { ownerFor } from "../tests/owner-rpc.ts"; * auto-allocation retry onto a known candidate, which needs the exact first port a real round trip * picked; a real TCP bind failure, which a probe-equipped `Network.ts` callback would otherwise * intercept first on a platform where overlapping binds succeed; and a stack-wide destroy's - * deferred row release through a real, in-process `Owner`, whose first attempt a concrete sweep + * retained row release through a real, in-process `Owner`, whose first attempt a concrete sweep * failure leaves uncertain. All exercise the real per-user registry and real sockets; see * `port-reservation.e2e.test.ts` for the end-to-end scenarios, and `Network.integration.test.ts`'s * "keeps shared routes independent and retains the shared claim" for the shared-listener @@ -54,7 +54,6 @@ const saveStack = (state: StackNamespace.Interface, root: string, id: string) => instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); /** Binds like a brand-new listener Network.ts would create: probed first, then really bound, so @@ -238,7 +237,7 @@ it.live( ); it.live( - "a stack-wide destroy defers a dedicated row's release; a cleanup failure keeps it uncertain, and the next destroy, once resolved, releases it", + "a stack-wide destroy retains a dedicated row; a cleanup failure keeps it, and the next destroy, once resolved, releases it", () => Effect.scoped( Effect.gen(function* () { @@ -254,7 +253,6 @@ it.live( instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }; yield* state.save(saved); @@ -314,7 +312,7 @@ it.live( const firstAttempt = yield* owner.namespace.destroy.pipe(Effect.exit); expect(firstAttempt._tag).toBe("Failure"); - // Deferred: the row survives a destroy that could not confirm nothing remains. + // Retained: the row survives a destroy that could not confirm nothing remains. expect(yield* portReservations.find(realStateRoot, id, `${mail.id}:http`)).toBeDefined(); resolved = true; diff --git a/packages/stack/src/Ports.ts b/packages/stack/src/Ports.ts index e2239aa658..a232141fd7 100644 --- a/packages/stack/src/Ports.ts +++ b/packages/stack/src/Ports.ts @@ -235,31 +235,18 @@ export const reserveNativePort = Effect.fn("Ports.reserveNativePort")( ); const resolveRequest = ( - stack: StackNamespace.SavedStack, request: PortRequest, owned: number | undefined, -): Effect.Effect< - { readonly saved: StackNamespace.PortClaim | undefined; readonly requested: number | "auto" }, - PortError -> => { - const saved = stack.ports.find((entry) => entry.key === request.key); - if (saved !== undefined && saved.host !== request.host) - return Effect.fail( - new PortError({ - key: request.key, - message: "The requested listener differs from its saved assignment", - }), - ); - const expected = owned ?? saved?.port; - if (request.port !== "auto" && expected !== undefined && expected !== request.port) +): Effect.Effect => { + if (request.port !== "auto" && owned !== undefined && owned !== request.port) return Effect.fail( new PortError({ key: request.key, message: "The requested listener differs from its reserved assignment", }), ); - const requested = expected ?? request.port; - if (requested === "auto") return Effect.succeed({ saved, requested }); + const requested = owned ?? request.port; + if (requested === "auto") return Effect.succeed(requested); if (!Number.isInteger(requested) || requested < 1 || requested > 65535) return Effect.fail(new PortError({ key: request.key, message: "Invalid public port" })); if (requested >= nativePortBase && requested < portBase) @@ -269,7 +256,7 @@ const resolveRequest = ( message: `Public port ${requested} is inside ${nativePortBase}-${portBase - 1}, which is reserved for native service ports; choose a port outside that range`, }), ); - return Effect.succeed({ saved, requested }); + return Effect.succeed(requested); }; /** @@ -336,7 +323,7 @@ export const makePorts = (state: StackNamespace.Interface) => if (stack === undefined) return yield* new PortError({ key: request.key, message: "Stack is not registered" }); const owned = yield* portReservations.find(stateRoot, request.stackId, request.key); - const { saved, requested } = yield* resolveRequest(stack, request, owned); + const requested = yield* resolveRequest(request, owned); const owner = yield* Scope.Scope; yield* Effect.annotateCurrentSpan({ "stack.endpoint": request.key, @@ -344,49 +331,32 @@ export const makePorts = (state: StackNamespace.Interface) => requested !== "auto" ? (owned !== undefined ? "owned" : "fixed") : "auto", }); - const publishIfNeeded = (port: number) => - saved === undefined || saved.port !== port || saved.host !== request.host - ? state.save({ - ...stack, - ports: [ - ...stack.ports.filter((entry) => entry.key !== request.key), - { key: request.key, host: request.host, port }, - ], - }) - : Effect.void; - - /** Binds `port` and publishes it, against whatever scope the caller provides. */ - const bindAndPublish = (port: number) => - Effect.gen(function* () { - const listener = yield* bind(request.host, port).pipe( - Effect.mapError( - (cause) => - new PortError({ - key: request.key, - message: `Cannot bind ${request.key} at ${request.host}:${port}: ${cause.message}`, - cause: cause.cause, - conflict: - cause.conflict ?? - (isAddressInUse(cause.cause) - ? { port, endpoint: request.key, holder: "foreign" as const } - : undefined), - }), - ), - ); - yield* publishIfNeeded(port); - return { port, listener }; - }); - - /** - * One bind attempt at `port`, in its own forked scope so a failure rolls back both the - * listener and the `state.json` publish together; a success holds the scope open instead. - */ + /** Binds `port` against whatever scope the caller provides. */ + const bindListener = (port: number) => + bind(request.host, port).pipe( + Effect.map((listener) => ({ port, listener })), + Effect.mapError( + (cause) => + new PortError({ + key: request.key, + message: `Cannot bind ${request.key} at ${request.host}:${port}: ${cause.message}`, + cause: cause.cause, + conflict: + cause.conflict ?? + (isAddressInUse(cause.cause) + ? { port, endpoint: request.key, holder: "foreign" as const } + : undefined), + }), + ), + ); + + /** One bind attempt at `port`, in its own forked scope so a failure closes the listener; a success holds the scope open instead. */ const attemptBind = (port: number) => Effect.uninterruptibleMask((restore) => Effect.gen(function* () { const scope = yield* Scope.fork(owner, "sequential"); return yield* restore( - bindAndPublish(port).pipe(Effect.provideService(Scope.Scope, scope)), + bindListener(port).pipe(Effect.provideService(Scope.Scope, scope)), ).pipe( Effect.onExit((exit) => Exit.isFailure(exit) ? Scope.close(scope, exit) : Effect.void, @@ -434,7 +404,7 @@ export const makePorts = (state: StackNamespace.Interface) => if (holder !== undefined) return undefined; const scope = yield* Scope.fork(owner, "sequential"); return yield* restore( - bindAndPublish(port).pipe(Effect.provideService(Scope.Scope, scope)), + bindListener(port).pipe(Effect.provideService(Scope.Scope, scope)), ).pipe( Effect.onExit((exit) => Exit.isFailure(exit) @@ -485,18 +455,13 @@ export const makePorts = (state: StackNamespace.Interface) => ); }); + /** The port this stack's reservation holds for `key`, or `undefined` when it holds none. */ + const assigned = Effect.fn("Ports.assigned")((stackId: string, key: string) => + portReservations.find(stateRoot, stackId, key), + ); + const release = Effect.fn("Ports.release")(function* (stackId: string, key: string) { - yield* state.withLock( - Effect.gen(function* () { - yield* portReservations.release(stateRoot, stackId, key); - const stack = yield* state.read(stackId); - if (stack !== undefined) - yield* state.save({ - ...stack, - ports: stack.ports.filter((entry) => entry.key !== key), - }); - }), - ); + yield* state.withLock(portReservations.release(stateRoot, stackId, key)); }); /** Releases every reservation a stack holds, regardless of endpoint; used at destroy's success boundary. */ @@ -504,5 +469,5 @@ export const makePorts = (state: StackNamespace.Interface) => yield* portReservations.releaseStack(stateRoot, stackId); }); - return { acquire, release, releaseStack }; + return { acquire, assigned, release, releaseStack }; }); diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index 6cd18960a7..a4c4ff7882 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -194,7 +194,6 @@ const startHost = (stateRoot: string, cacheRoot: string, stackId: string, projec instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); return yield* launchHost(state, { stateRoot, cacheRoot, stackId }); }); @@ -352,7 +351,6 @@ it.live.skipIf(process.platform === "win32")( instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); yield* stateB3.save({ id: stackId, @@ -365,7 +363,6 @@ it.live.skipIf(process.platform === "win32")( instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const accessA3 = yield* launchHost(stateA3, { stateRoot: rootA, @@ -463,7 +460,6 @@ it.live.skipIf(process.platform === "win32")( instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); // The shortened poll interval comes only from this dedicated test entrypoint, through @@ -559,7 +555,6 @@ const abandonsWhileStopDrains = (trigger: StopTrigger) => instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); const access = yield* launchHost(state, { @@ -673,7 +668,6 @@ it.live.skipIf(process.platform === "win32")( instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }); yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); const access = yield* launchHost(state, { @@ -751,7 +745,6 @@ it.live.skipIf(process.platform === "win32")( instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); const access = yield* launchHost(state, { @@ -871,7 +864,6 @@ it.live.skipIf(process.platform === "win32")( instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); const access = yield* launchHost(state, { @@ -949,7 +941,6 @@ it.live.skipIf(process.platform === "win32")( instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); const access = yield* launchHost(state, { diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts index 0c4df16eb3..5bbb41d780 100644 --- a/packages/stack/src/StackHost.integration.test.ts +++ b/packages/stack/src/StackHost.integration.test.ts @@ -322,7 +322,6 @@ it.live("preserves composition outcomes over RPC", () => instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }; yield* state.save(saved); const owner = yield* ownerFor({ @@ -378,7 +377,6 @@ it.live("stops serving after a failed shutdown and keeps reporting that failure" instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }; yield* state.save(saved); const owner = yield* ownerFor({ @@ -428,7 +426,6 @@ it.live( instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }; yield* state.save(saved); const owner = yield* ownerFor({ @@ -487,7 +484,6 @@ it.live("reports a destroy failure and the owner exits", () => instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }; yield* state.save(saved); const owner = yield* ownerFor({ @@ -551,7 +547,6 @@ it.live("rejects destroy while stop is in flight", () => instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }; yield* state.save(saved); const owner = yield* ownerFor({ @@ -597,7 +592,6 @@ it.live("exits after a namespace shutdown defect without rerunning cleanup", () instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }; yield* state.save(saved); const owner = yield* ownerFor({ @@ -643,7 +637,6 @@ it.live( instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const access = yield* launchHost(state, { stateRoot: `${root}/state`, @@ -842,7 +835,6 @@ it.live("finishes detached shutdown after the caller disconnects", () => instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const owner = yield* ownerFor({ saved: { @@ -852,7 +844,6 @@ it.live("finishes detached shutdown after the caller disconnects", () => instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }, state, root: `${root}/data`, @@ -894,7 +885,6 @@ it.live("withdraws a command waiting for its prerequisite", () => instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }; yield* state.save(saved); const owner = yield* ownerFor({ @@ -953,7 +943,6 @@ const disconnectFixture = (prefix: string) => instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }; yield* state.save(saved); return { root, state, saved }; @@ -1211,7 +1200,6 @@ it.live( instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const access = yield* launchHost(state, { stateRoot, diff --git a/packages/stack/src/StackNamespace.ts b/packages/stack/src/StackNamespace.ts index fa4e51a633..2197ec4755 100644 --- a/packages/stack/src/StackNamespace.ts +++ b/packages/stack/src/StackNamespace.ts @@ -8,7 +8,6 @@ export { NamespaceError } from "./namespace/Capabilities.ts"; export { LeaseHeldError, LeaseHolder } from "./namespace/Lease.ts"; export { resolveStackDataRoot } from "./namespace/Paths.ts"; export { - PortClaim, SavedStack, StackCredentials, StackKeysInput, diff --git a/packages/stack/src/Sweep.integration.test.ts b/packages/stack/src/Sweep.integration.test.ts index 3e58be9bf7..3dd45457dc 100644 --- a/packages/stack/src/Sweep.integration.test.ts +++ b/packages/stack/src/Sweep.integration.test.ts @@ -149,7 +149,6 @@ const saved = ( identity: { projectRoot, branchContext: "sweep-test", stackName: id }, instances: [], composition: { members: [], dependencies: [] }, - ports: [], }); it.live.skipIf(process.platform === "win32")( diff --git a/packages/stack/src/composition/Supabase.native.integration.test.ts b/packages/stack/src/composition/Supabase.native.integration.test.ts index 39bd792e11..21f42b67c7 100644 --- a/packages/stack/src/composition/Supabase.native.integration.test.ts +++ b/packages/stack/src/composition/Supabase.native.integration.test.ts @@ -60,7 +60,6 @@ const initial = (id: string): SavedStack => ({ instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); it.live("reports the failed binding and cleanup reason with the retained instance ID", () => diff --git a/packages/stack/src/composition/Supabase.unit.test.ts b/packages/stack/src/composition/Supabase.unit.test.ts index 6fb5e2a36e..54f17904d2 100644 --- a/packages/stack/src/composition/Supabase.unit.test.ts +++ b/packages/stack/src/composition/Supabase.unit.test.ts @@ -43,6 +43,7 @@ const makeInstance = ( close: Effect.void, confirmRemoved: Effect.void, release: Effect.void, + releasePorts: Effect.void, hasEndpoint: options.hasEndpoint ?? true, inputs: options.inputs ?? [], outputs: options.outputs ?? {}, diff --git a/packages/stack/src/effect.integration.test.ts b/packages/stack/src/effect.integration.test.ts index 722ed319a2..d2392eda1a 100644 --- a/packages/stack/src/effect.integration.test.ts +++ b/packages/stack/src/effect.integration.test.ts @@ -577,7 +577,6 @@ it.live("reclaims a registered stack's leftovers when stopping without an owner" runtime: "native", instances: [{ id: "leftover", creation: { service: "mail", config: {} } }], composition: { members: [], dependencies: [] }, - ports: [], }); const leftover = `${options.stateRoot}/${id}/data/leftover`; yield* fs.makeDirectory(leftover, { recursive: true }); @@ -842,7 +841,6 @@ it.live("replaces a dead session stack that holds the requested identity", () => runtime: "native", instances: [{ id: "abandoned", creation: { service: "mail", config: {} } }], composition: { members: [], dependencies: [] }, - ports: [], }); yield* Effect.scoped( diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index d5b2258129..0fbfa54819 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -857,7 +857,6 @@ export const create = Effect.fn("Stack.create")( runtime: options.runtime, instances: [], composition: { members: [], dependencies: [] }, - ports: [], }; const locations = { stateRoot: options.stateRoot, cacheRoot: options.cacheRoot }; const existing = yield* state.read(id); diff --git a/packages/stack/src/internal/dispatch.integration.test.ts b/packages/stack/src/internal/dispatch.integration.test.ts index 61364b2536..c306ab2434 100644 --- a/packages/stack/src/internal/dispatch.integration.test.ts +++ b/packages/stack/src/internal/dispatch.integration.test.ts @@ -66,7 +66,6 @@ it.live("dispatches compiled owner and native launchers through the production b instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const runOwner = (mode: "owner" | "stop") => diff --git a/packages/stack/src/lifetime.integration.test.ts b/packages/stack/src/lifetime.integration.test.ts index b2b692bc24..7710523eb0 100644 --- a/packages/stack/src/lifetime.integration.test.ts +++ b/packages/stack/src/lifetime.integration.test.ts @@ -124,7 +124,6 @@ it.live.skipIf(process.platform === "win32")( )(ready); const state = yield* stateFor(stateRoot); expect(yield* state.leased(stackId)).toBe(true); - expect((yield* state.read(stackId))?.ports.length).toBeGreaterThan(0); const ownerPid = yield* captureOwnerPid({ stateRoot, cacheRoot }, stackId); const owned = yield* descendantsOf(ownerPid); expect(owned.length, "the owner runs the native mail service").toBeGreaterThan(0); @@ -220,7 +219,6 @@ it.live.skipIf(process.platform === "win32")( instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); // The shortened poll interval comes only from this dedicated test entrypoint, through // the internal `Context.Reference`; production startup never reads an env var or `Config`. @@ -275,7 +273,6 @@ it.live.skipIf(process.platform === "win32")( instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const reclaimedAccess = yield* launchHost(reclaimedState, { stateRoot: reclaimedStateRoot, diff --git a/packages/stack/src/namespace/Namespace.e2e.test.ts b/packages/stack/src/namespace/Namespace.e2e.test.ts index 77068e4bd5..d55859c5cd 100644 --- a/packages/stack/src/namespace/Namespace.e2e.test.ts +++ b/packages/stack/src/namespace/Namespace.e2e.test.ts @@ -55,7 +55,6 @@ const savedStack = (root: string, id: string): StackNamespace.SavedStack => ({ instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); it.live( diff --git a/packages/stack/src/namespace/Namespace.integration.test.ts b/packages/stack/src/namespace/Namespace.integration.test.ts index 2610200f58..9e1beca083 100644 --- a/packages/stack/src/namespace/Namespace.integration.test.ts +++ b/packages/stack/src/namespace/Namespace.integration.test.ts @@ -40,7 +40,6 @@ const initial: StackNamespace.SavedStack = { instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }; const run = (effect: Effect.Effect) => diff --git a/packages/stack/src/namespace/Namespace.windows.e2e.test.ts b/packages/stack/src/namespace/Namespace.windows.e2e.test.ts index 649d60c02a..acedc44d4a 100644 --- a/packages/stack/src/namespace/Namespace.windows.e2e.test.ts +++ b/packages/stack/src/namespace/Namespace.windows.e2e.test.ts @@ -22,7 +22,6 @@ const saved: StackNamespace.SavedStack = { instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }; class HolderError extends Data.TaggedError("NamespaceWindowsHolderError")<{ diff --git a/packages/stack/src/namespace/Registry.ts b/packages/stack/src/namespace/Registry.ts index e33c345037..a34f45cd3e 100644 --- a/packages/stack/src/namespace/Registry.ts +++ b/packages/stack/src/namespace/Registry.ts @@ -49,13 +49,6 @@ export const StackCredentials = Schema.Struct({ }); export interface StackCredentials extends Schema.Schema.Type {} -export const PortClaim = Schema.Struct({ - key: Schema.String, - host: Schema.String, - port: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 65535 })), -}); -export interface PortClaim extends Schema.Schema.Type {} - /** A session stack is destroyed when its creator exits; a detached stack outlives it. */ export const StackLifetime = Schema.Literals(["session", "detached"]); export type StackLifetime = Schema.Schema.Type; @@ -78,7 +71,6 @@ export const SavedStack = Schema.Struct({ ), composition: CompositionConfig, credentials: Schema.optionalKey(StackCredentials), - ports: Schema.Array(PortClaim), }); export interface SavedStack extends Schema.Schema.Type {} diff --git a/packages/stack/src/port-reservation.e2e.test.ts b/packages/stack/src/port-reservation.e2e.test.ts index e9ef14ab88..3498ed077f 100644 --- a/packages/stack/src/port-reservation.e2e.test.ts +++ b/packages/stack/src/port-reservation.e2e.test.ts @@ -5,7 +5,6 @@ import * as Net from "node:net"; import { create, open, type Observation } from "./effect.ts"; import { ownerExitProbe, waitForOwnerExit } from "./HostProcess.ts"; import * as PortReservations from "./namespace/PortReservations.ts"; -import * as StackNamespace from "./StackNamespace.ts"; import { captureOwnerPid } from "../tests/owner.ts"; import { destroyTestStack } from "../tests/stack-cleanup.ts"; @@ -333,81 +332,6 @@ it.live( { timeout: 120_000 }, ); -it.live( - "a restart republishes state.json when the registry already holds a port its saved claim forgot", - () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const cacheRoot = yield* fs.makeTempDirectoryScoped({ - prefix: "port-reservation-desync-cache-", - }); - const { stack: stackA, stateRoot: stateRootA } = yield* makeStack(fs, cacheRoot); - const mailA = yield* stackA.services.create(mailOn("auto")); - yield* mailA.start; - yield* mailA.ready; - const port = portOf(yield* mailA.status); - yield* stackA.stop; - - // Simulate a crash between the registry commit and the local `state.json` publish: the - // registry still names this port, but the saved claim that mirrors it is gone. - const state = yield* StackNamespace.Service.pipe( - Effect.provide(StackNamespace.layer({ root: stateRootA })), - ); - const saved = yield* state.read(stackA.id); - if (saved === undefined) return yield* Effect.die("stack state missing"); - yield* state.save({ ...saved, ports: [] }); - - const reopenedA = yield* open({ id: stackA.id, stateRoot: stateRootA, cacheRoot }); - const reopenedMailA = yield* reopenedA.services.get(mailA.id); - yield* reopenedMailA.start; - yield* reopenedMailA.ready; - expect(portOf(yield* reopenedMailA.status)).toBe(port); - expect((yield* state.read(stackA.id))?.ports.some((entry) => entry.port === port)).toBe( - true, - ); - }), - ), - { timeout: 120_000 }, -); - -it.live( - "a restart re-establishes a registry row its saved claim remembers but the registry lost", - () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const cacheRoot = yield* fs.makeTempDirectoryScoped({ - prefix: "port-reservation-missing-row-cache-", - }); - const { stack: stackA, stateRoot: stateRootA } = yield* makeStack(fs, cacheRoot); - const mailA = yield* stackA.services.create(mailOn("auto")); - yield* mailA.start; - yield* mailA.ready; - const port = portOf(yield* mailA.status); - yield* stackA.stop; - - // Simulate the registry row disappearing independently of `state.json` (for example, a - // registry file replaced out from under a stopped stack): the saved claim still names the - // port, but no row backs it. - const realStateRoot = yield* fs.realPath(stateRootA); - const reservationContext = yield* Layer.build(PortReservations.layer); - const portReservations = Context.get(reservationContext, PortReservations.Service); - yield* portReservations.release(realStateRoot, stackA.id, `${mailA.id}:http`); - - const reopenedA = yield* open({ id: stackA.id, stateRoot: stateRootA, cacheRoot }); - const reopenedMailA = yield* reopenedA.services.get(mailA.id); - yield* reopenedMailA.start; - yield* reopenedMailA.ready; - expect(portOf(yield* reopenedMailA.status)).toBe(port); - expect(yield* portReservations.find(realStateRoot, stackA.id, `${mailA.id}:http`)).toBe( - port, - ); - }), - ), - { timeout: 120_000 }, -); - it.live( "composing a member on an already-claimed port surfaces a structured conflict naming the holder", () => @@ -479,10 +403,6 @@ it.live( expect(failure.message).toContain("10000-19999"); expect(failure.message).toContain("choose a port outside that range"); - const state = yield* StackNamespace.Service.pipe( - Effect.provide(StackNamespace.layer({ root: stateRoot })), - ); - expect((yield* state.read(stack.id))?.ports).toEqual([]); const reservationContext = yield* Layer.build(PortReservations.layer); const portReservations = Context.get(reservationContext, PortReservations.Service); expect( diff --git a/packages/stack/src/runtime/Container.integration.test.ts b/packages/stack/src/runtime/Container.integration.test.ts index 4a3bb724fb..1129807967 100644 --- a/packages/stack/src/runtime/Container.integration.test.ts +++ b/packages/stack/src/runtime/Container.integration.test.ts @@ -1659,7 +1659,6 @@ describe("container process adapter", () => { instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }; yield* state.save(saved); yield* Owner.sweepContainers(saved, root, target).pipe( diff --git a/packages/stack/src/services/Rest.integration.test.ts b/packages/stack/src/services/Rest.integration.test.ts index eef28b0cad..b465d68cac 100644 --- a/packages/stack/src/services/Rest.integration.test.ts +++ b/packages/stack/src/services/Rest.integration.test.ts @@ -62,7 +62,6 @@ describe("service catalog", () => { instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const network = yield* makeTestNetwork({ stackId, runtime: "docker", state }); const secret = "catalog-rest-secret-with-at-least-32-chars"; @@ -194,7 +193,7 @@ describe("service catalog", () => { yield* Ref.set(databaseActive, false); yield* databaseNamespace.close; yield* databaseNamespace.release; - yield* network.release; + yield* network.releaseStack; }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 120_000 }, diff --git a/packages/stack/src/services/StorageGateway.integration.test.ts b/packages/stack/src/services/StorageGateway.integration.test.ts index 2813cbbb22..1e60902fc1 100644 --- a/packages/stack/src/services/StorageGateway.integration.test.ts +++ b/packages/stack/src/services/StorageGateway.integration.test.ts @@ -58,7 +58,6 @@ const serveStorage = Effect.fnUntraced(function* (runtime: SavedStack["runtime"] instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }; const state = Context.get( yield* Layer.build(StackNamespace.layer({ root: stateRoot })), diff --git a/packages/stack/src/shutdown.integration.test.ts b/packages/stack/src/shutdown.integration.test.ts index 97da9b095a..f9f3d7a8a6 100644 --- a/packages/stack/src/shutdown.integration.test.ts +++ b/packages/stack/src/shutdown.integration.test.ts @@ -60,7 +60,6 @@ const withHeldOwner = ( instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; return yield* Effect.acquireUseRelease( diff --git a/packages/stack/tests/standalone-service.ts b/packages/stack/tests/standalone-service.ts index 031a7233e0..fdd0cc428e 100644 --- a/packages/stack/tests/standalone-service.ts +++ b/packages/stack/tests/standalone-service.ts @@ -36,6 +36,7 @@ export const makeStandaloneService = Effect.fn("Test.makeStandaloneService")(fun close: Effect.void, confirmRemoved: Effect.void, release: Effect.void, + releasePorts: Effect.void, hasEndpoint: true, inputs: [], outputs: {}, From 74d92cdbd040aedb192a48794f2a2f630054a0cd Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 01:48:45 +0200 Subject: [PATCH 067/142] fix(stack): remove the stack registration before releasing its port reservations --- packages/stack/ARCHITECTURE.md | 2 +- packages/stack/README.md | 2 +- packages/stack/src/Owner.integration.test.ts | 39 ++++++----- packages/stack/src/Owner.ts | 12 +++- packages/stack/src/Ports.integration.test.ts | 71 ++++++++++++++++++++ 5 files changed, 101 insertions(+), 25 deletions(-) diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 24c4c81598..efd9fef890 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -357,7 +357,7 @@ flowchart TB Stopped -->|explicit start| Start["Orchestrator starts selected services
Reuses saved public ports"] ``` -Sleep retains the public listener needed to wake. Whole-stack stop closes listeners while preserving the registry's reservations; the next start rebinds the same ports. Destroying an individual instance deletes only the registry rows it owns, not unrelated services' shared listeners or rows; a stack-wide destroy deletes its rows only after it fully succeeds, and a failed destroy keeps them. +Sleep retains the public listener needed to wake. Whole-stack stop closes listeners while preserving the registry's reservations; the next start rebinds the same ports. Destroying an individual instance deletes only the registry rows it owns, not unrelated services' shared listeners or rows; a stack-wide destroy deletes its rows only after the registration is removed, so a failed destroy keeps them and a failed release leaves them for lazy reclamation. Composition validation permits lazy activation only for instances with a configured public wake endpoint. A route-less prerequisite, such as pg-meta without its own public endpoint, is eager; it is not implicitly armed through a dependent. This avoids a second wake-permission mechanism. Internal sleep is available only for lazy instances with a supported public wake route and an enabled idle policy. The default composition gives every lazy service one, Functions included; the database is eager. Functions inspector access remains possible while health is starting; ordinary application traffic waits for healthy. diff --git a/packages/stack/README.md b/packages/stack/README.md index f7cf7ebd1e..8c32e93d06 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -104,7 +104,7 @@ field names the port, the endpoint, and either the live holder (`{ stackId, stat `"foreign"` for a process outside the registry. A fixed or previously saved port is never silently reassigned; only automatic allocation tries another candidate. A reservation is released when deleting an individual service releases the endpoints only it owned, or when a stack-wide `destroy` -has fully succeeded; a failed `destroy` keeps every reservation, and stopping a stack or killing its +has removed the registration; a failed `destroy` keeps every reservation, and stopping a stack or killing its owner leaves them in place. A reservation whose owning stack's `state.json` is confirmed gone (`ENOENT`, for example after deleting its state root) is reclaimed lazily by the next stack that needs its port. diff --git a/packages/stack/src/Owner.integration.test.ts b/packages/stack/src/Owner.integration.test.ts index daab355b60..9a83aac599 100644 --- a/packages/stack/src/Owner.integration.test.ts +++ b/packages/stack/src/Owner.integration.test.ts @@ -13,7 +13,7 @@ import * as PortReservations from "./namespace/PortReservations.ts"; import * as StackNamespace from "./StackNamespace.ts"; import type { SavedStack } from "./StackNamespace.ts"; import { DEFAULT_LOCAL_JWT_SECRET } from "./Defaults.ts"; -import { ServiceCreation, type ServiceCreationInput } from "./services/Catalog.ts"; +import type { ServiceCreationInput } from "./services/Catalog.ts"; import { ownerFor } from "../tests/owner-rpc.ts"; import { dockerEngineTarget } from "../tests/engine-target.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; @@ -784,35 +784,34 @@ it.effect( const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-duplicate-" }); const state = yield* stateFor(`${root}/state`); - const creation = yield* Schema.decodeEffect(ServiceCreation)({ - service: "mail", - config: {}, - endpoints: { http: { port: "auto" } }, - }); - const instance = { id: "mail", creation }; - const stack: SavedStack = { - ...initial(`owner-duplicate-${randomUUID().slice(0, 8)}`), - instances: [instance], - }; + const stack = initial(`owner-duplicate-${randomUUID().slice(0, 8)}`); yield* state.save(stack); + const owner = yield* ownerFor({ saved: stack, state, root: `${root}/data`, cacheRoot }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + const definitions = yield* owner.rpc.supabaseComposition({ + services: [{ service: "mail", config: {}, endpoints: { http: { port: "auto" } } }], + }); + const mail = definitions.find((entry) => entry.creation.service === "mail"); + if (mail === undefined) return yield* Effect.die("mail missing from the composition"); const realStateRoot = yield* fs.realPath(`${root}/state`); const portReservations = yield* PortReservations.Service; - expect(yield* portReservations.reserve(realStateRoot, stack.id, "mail:http", 54_321)).toBe( - undefined, - ); - yield* Effect.addFinalizer(() => - portReservations.release(realStateRoot, stack.id, "mail:http").pipe(Effect.ignore), - ); + const endpoint = `${mail.id}:http`; + const assigned = yield* portReservations.find(realStateRoot, stack.id, endpoint); + expect(assigned).toBeDefined(); + const saved = yield* state.read(stack.id); + if (saved === undefined) return yield* Effect.die("saved state disappeared"); + const instance = saved.instances.find((entry) => entry.id === mail.id); + if (instance === undefined) return yield* Effect.die("mail instance was not saved"); const failure = yield* ownerFor({ - saved: { ...stack, instances: [instance, instance] }, + saved: { ...saved, instances: [instance, instance] }, state, root: `${root}/data`, cacheRoot, }).pipe(Effect.flip); - expect(failure.message).toBe("Duplicate instance mail"); - expect(yield* portReservations.find(realStateRoot, stack.id, "mail:http")).toBe(54_321); + expect(failure.message).toBe(`Duplicate instance ${mail.id}`); + expect(yield* portReservations.find(realStateRoot, stack.id, endpoint)).toBe(assigned); }), ).pipe( Effect.provide( diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index e175adb768..9aca1d23fc 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -795,14 +795,20 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { definitionGate.withPermits(1), Effect.withSpan("Owner.stopNamespace"), ), - // Every instance's teardown closes its listeners but retains its saved port assignments; - // `releaseStack` drops them together, only once destroy has fully succeeded. + // Instance teardown retains port rows until the registration is gone; the release after + // that is best-effort because `isGone` reclaims a gone stack's rows lazily. destroy: network.drain.pipe( Effect.andThen(orchestrator.destroyNamespace), Effect.andThen(sweep), Effect.andThen(removeContainerEnvRoot), - Effect.andThen(network.releaseStack), Effect.andThen(options.state.remove(stackId)), + Effect.andThen( + network.releaseStack.pipe( + Effect.catch((cause) => + Effect.logWarning("Destroyed stack could not release its port reservations", cause), + ), + ), + ), definitionGate.withPermits(1), Effect.withSpan("Owner.destroyNamespace"), ), diff --git a/packages/stack/src/Ports.integration.test.ts b/packages/stack/src/Ports.integration.test.ts index 3d40882249..2ec9f0b23e 100644 --- a/packages/stack/src/Ports.integration.test.ts +++ b/packages/stack/src/Ports.integration.test.ts @@ -332,6 +332,77 @@ it.live( ), ); +it.live("a destroy that cannot remove the registration keeps every reservation", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-remove-fails-" }); + const id = `destroy-remove-fails-${randomUUID()}`; + const state = yield* makeTestState(`${root}/state`); + const saved = { + id, + runtime: "native" as const, + identity: { projectRoot: root, branchContext: "test", stackName: id }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + }; + yield* state.save(saved); + let removable = false; + const unremovableState: StackNamespace.Interface = { + ...state, + remove: (target) => + removable + ? state.remove(target) + : Effect.fail( + new StackNamespace.NamespaceError({ + operation: "remove", + message: "injected failure", + }), + ), + }; + const owner = yield* ownerFor({ + saved, + state: unremovableState, + root: `${root}/data`, + cacheRoot: `${root}/cache`, + }); + yield* Effect.addFinalizer(() => + Effect.suspend(() => { + removable = true; + return owner.namespace.destroy; + }).pipe(Effect.ignore), + ); + const definitions = yield* owner.rpc.supabaseComposition({ + services: [{ service: "mail", config: {}, endpoints: { http: { port: "auto" } } }], + }); + const mail = definitions.find((entry) => entry.creation.service === "mail"); + if (mail === undefined) return yield* Effect.die("mail missing from the composition"); + const realStateRoot = yield* fs.realPath(`${root}/state`); + const portReservations = yield* PortReservations.Service.pipe( + Effect.provide(PortReservations.layer), + ); + const assigned = yield* portReservations.find(realStateRoot, id, `${mail.id}:http`); + expect(assigned).toBeDefined(); + + const failed = yield* owner.namespace.destroy.pipe(Effect.exit); + + expect(failed._tag).toBe("Failure"); + expect(yield* portReservations.find(realStateRoot, id, `${mail.id}:http`)).toBe(assigned); + }), + ).pipe( + Effect.provide( + Layer.merge( + NodeServices.layer, + Layer.merge( + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), + ), +); + it.live("reserveNativePort skips a port a wildcard listener already holds", () => Effect.scoped( Effect.gen(function* () { From 13d224cc52b38e8dc858111c199ae3c27e687b32 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 01:51:56 +0200 Subject: [PATCH 068/142] test(stack): keep port registry fixtures scoped and let cleanup reach the real destroy --- packages/stack/src/Owner.integration.test.ts | 10 +++--- packages/stack/src/Ports.integration.test.ts | 33 +++++++++++--------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/packages/stack/src/Owner.integration.test.ts b/packages/stack/src/Owner.integration.test.ts index 9a83aac599..33e8fe30e8 100644 --- a/packages/stack/src/Owner.integration.test.ts +++ b/packages/stack/src/Owner.integration.test.ts @@ -977,8 +977,9 @@ it.live( yield* owner.rpc.readyService({ id: created.id }); const realStateRoot = yield* fs.realPath(`${tempRoot}/state`); - const portReservations = yield* PortReservations.Service.pipe( - Effect.provide(PortReservations.layer), + const portReservations = Context.get( + yield* Layer.build(PortReservations.layer), + PortReservations.Service, ); expect( yield* portReservations.find(realStateRoot, stackId, `${created.id}:sql`), @@ -1046,8 +1047,9 @@ it.live( yield* owner.rpc.readyService({ id: created.id }); const realStateRoot = yield* fs.realPath(`${tempRoot}/state`); - const portReservations = yield* PortReservations.Service.pipe( - Effect.provide(PortReservations.layer), + const portReservations = Context.get( + yield* Layer.build(PortReservations.layer), + PortReservations.Service, ); expect( yield* portReservations.find(realStateRoot, stackId, `${created.id}:sql`), diff --git a/packages/stack/src/Ports.integration.test.ts b/packages/stack/src/Ports.integration.test.ts index 2ec9f0b23e..f755e9ab06 100644 --- a/packages/stack/src/Ports.integration.test.ts +++ b/packages/stack/src/Ports.integration.test.ts @@ -204,8 +204,9 @@ it.live( yield* saveStack(state, root, id); const ports = yield* makePorts(state); const realRoot = yield* fs.realPath(root); - const portReservations = yield* PortReservations.Service.pipe( - Effect.provide(PortReservations.layer), + const portReservations = Context.get( + yield* Layer.build(PortReservations.layer), + PortReservations.Service, ); // Stands in for a probe or bind that never settles: the first candidate's row is @@ -305,8 +306,9 @@ it.live( if (mail === undefined) return yield* Effect.die("mail missing from the composition"); const realStateRoot = yield* fs.realPath(`${root}/state`); - const portReservations = yield* PortReservations.Service.pipe( - Effect.provide(PortReservations.layer), + const portReservations = Context.get( + yield* Layer.build(PortReservations.layer), + PortReservations.Service, ); expect(yield* portReservations.find(realStateRoot, id, `${mail.id}:http`)).toBeDefined(); @@ -352,14 +354,16 @@ it.live("a destroy that cannot remove the registration keeps every reservation", const unremovableState: StackNamespace.Interface = { ...state, remove: (target) => - removable - ? state.remove(target) - : Effect.fail( - new StackNamespace.NamespaceError({ - operation: "remove", - message: "injected failure", - }), - ), + Effect.suspend(() => + removable + ? state.remove(target) + : Effect.fail( + new StackNamespace.NamespaceError({ + operation: "remove", + message: "injected failure", + }), + ), + ), }; const owner = yield* ownerFor({ saved, @@ -379,8 +383,9 @@ it.live("a destroy that cannot remove the registration keeps every reservation", const mail = definitions.find((entry) => entry.creation.service === "mail"); if (mail === undefined) return yield* Effect.die("mail missing from the composition"); const realStateRoot = yield* fs.realPath(`${root}/state`); - const portReservations = yield* PortReservations.Service.pipe( - Effect.provide(PortReservations.layer), + const portReservations = Context.get( + yield* Layer.build(PortReservations.layer), + PortReservations.Service, ); const assigned = yield* portReservations.find(realStateRoot, id, `${mail.id}:http`); expect(assigned).toBeDefined(); From 6d5ee55f0bbb2dcf7adabf866b84b1d4bb7a2599 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 02:00:49 +0200 Subject: [PATCH 069/142] refactor(stack): enforce the waiter budget with one timeout and name waiter kinds --- packages/stack/ARCHITECTURE.md | 3 +- packages/stack/src/Lifecycle.ts | 142 +++-------- packages/stack/src/Lifecycle.unit.test.ts | 221 +++++++----------- .../src/Orchestrator.integration.test.ts | 56 +++++ packages/stack/src/Orchestrator.ts | 147 ++++++------ packages/stack/src/Owner.ts | 8 +- packages/stack/src/composition/Supabase.ts | 4 +- 7 files changed, 253 insertions(+), 328 deletions(-) diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index efd9fef890..2fd2170142 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -240,7 +240,6 @@ Each owner holds one pure reducer, `Lifecycle.ts`: `reduce(state, event, now)` r | ----------------------------------- | -------------------------------------------------------------------- | | `Launch`, `Stop`, `Reprobe` | A fiber in the owner scope driving that service's execution | | `ArmIdleTimer`, `ArmCooldownTimer` | A timer that reports `IdleElapsed` or `CooldownElapsed` | -| `ArmWaiterTimeout` | The waiting caller's own deadline, which reports `WaiterExpired` | | `AdmitConnection`, `FailConnection` | Completing that waiter, after the state it was decided on is visible | | `RequestRejected` | The requesting caller's failure | @@ -262,7 +261,7 @@ A generation whose cleanup failed stays `stopping` with its cleanup failure reco | Waiting for readiness or following logs | End the observation without changing lifecycle | | Running an attached command invocation | Terminate and clean up that invocation | -Traffic waits carry a lease and a wake budget of 120 seconds by default, counted from the client's arrival, including any wait for the gate. At most 256 traffic waiters queue per service; beyond that, admission fails at once. A budget that expires fails only that waiter with the stage it was blocked on; the shared launch continues. Explicit operations (start, ready, restart, composition start) wait without a budget or cap. They fail when their target or a prerequisite fails, loses readiness or fails its cleanup. +Each waiter has a kind. A `traffic` waiter carries a lease and holds through a breaker cooldown. An `explicit` waiter (start, ready, restart, composition start) opens no lease and fails fast with its target's or a prerequisite's outcome when that fails, loses readiness or fails its cleanup; it waits without a budget or cap. The reducer holds no clocks for waiters. The orchestrator enforces the wake budget, 120 seconds by default, with one timeout around the whole traffic wait, from the client's arrival through the gate wait and the wait for readiness. On expiry it withdraws the waiter, which releases any lease already granted, and fails only that caller with the stage it was blocked on; the shared launch continues. At most 256 traffic waiters queue per service; beyond that, admission fails at once. A service is stopped by an explicit stop, never by a disconnected caller. diff --git a/packages/stack/src/Lifecycle.ts b/packages/stack/src/Lifecycle.ts index cbbf2f8851..231e767638 100644 --- a/packages/stack/src/Lifecycle.ts +++ b/packages/stack/src/Lifecycle.ts @@ -1,7 +1,5 @@ import { Data } from "effect"; -/** The wake budget every waiter gets. */ -export const waiterBudgetMillis = 120_000; const waiterCap = 256; const breakerThreshold = 3; const breakerCooldownMillis = 30_000; @@ -105,20 +103,22 @@ const initialBreaker: BreakerState = { interface Waiter { readonly id: number; - /** Undefined for an explicit operation's wait, which has no wake budget. */ - readonly deadline: number | undefined; + /** + * A traffic wait holds a lease and counts toward the cap; an explicit operation's wait only + * observes readiness and fails with its prerequisite's outcome. + */ + readonly kind: "traffic" | "explicit"; /** False for an acquisition (such as the Functions inspector) that bypasses target readiness. */ readonly requireReady: boolean; } /** Whether admission opens a work lease; an explicit operation's wait only observes readiness. */ -const isLeaseWaiter = (waiter: Waiter): boolean => waiter.deadline !== undefined; +const isLeaseWaiter = (waiter: Waiter): boolean => waiter.kind === "traffic"; export interface ServiceState { readonly intent: Intent; readonly phase: Phase; readonly leases: number; - readonly lastActivity: number; readonly idleEpoch: number; /** The epoch an outstanding idle timer was armed with, or undefined while none is armed. */ readonly idleArmedEpoch: number | undefined; @@ -147,11 +147,10 @@ export interface ServiceState { readonly cleanupFailure: { readonly cause: unknown } | undefined; } -const initialService = (spec: ServiceSpec, now: number): ServiceState => ({ +const initialService = (spec: ServiceSpec): ServiceState => ({ intent: spec.activation, phase: Phase.Stopped(), leases: 0, - lastActivity: now, idleEpoch: 0, idleArmedEpoch: undefined, breaker: initialBreaker, @@ -174,30 +173,26 @@ export interface LifecycleState { readonly generationCounters: ReadonlyMap; } -export const initialState = (graph: LifecycleGraph, now: number): LifecycleState => ({ +export const initialState = (graph: LifecycleGraph): LifecycleState => ({ graph, services: new Map( [...graph.services.values()].map((spec): [string, ServiceState] => [ spec.id, - initialService(spec, now), + initialService(spec), ]), ), generationCounters: new Map([...graph.services.keys()].map((id) => [id, 1])), }); export type LifecycleEvent = Data.TaggedEnum<{ - /** - * `requireReady: false` lets an acquisition (the Functions inspector) bypass target readiness. - * `openedAt` starts the wake budget when the client arrived, not when the gate admitted it. - */ + /** `requireReady: false` lets an acquisition (the Functions inspector) bypass target readiness. */ ConnectionOpened: { readonly id: string; readonly waiterId: number; readonly requireReady: boolean; - readonly openedAt?: number; }; ConnectionClosed: { readonly id: string }; - /** An explicit operation waiting for a session (or readiness) with no budget and no lease. */ + /** An explicit operation waiting for a session (or readiness) with no lease. */ ReadinessAwaited: { readonly id: string; readonly waiterId: number; @@ -241,7 +236,6 @@ export type LifecycleEvent = Data.TaggedEnum<{ DestroyRequested: { readonly id: string }; /** A destroy gives up its request or reservation, whether or not it removed the data. */ DestroyReleased: { readonly id: string }; - WaiterExpired: { readonly id: string; readonly waiterId: number }; WaiterCancelled: { readonly id: string; readonly waiterId: number }; /** * Atomically swaps the graph while preserving every continuing service's generation and epoch. @@ -260,7 +254,6 @@ export type LifecycleCommand = Data.TaggedEnum<{ readonly epoch: number; readonly delayMillis: number; }; - ArmWaiterTimeout: { readonly id: string; readonly waiterId: number; readonly deadline: number }; /** Schedules the `CooldownElapsed` event once the breaker's cooldown elapses. */ ArmCooldownTimer: { readonly id: string; @@ -387,7 +380,7 @@ const sessionReady = (state: LifecycleState, id: string): boolean => { const isBreakerOpen = (breaker: BreakerState): boolean => breaker.openUntil !== undefined; /** Names what a pending waiter is still blocked on, for a budget-expiry error. */ -const blockingStage = (state: LifecycleState, id: string): string => { +export const blockingStage = (state: LifecycleState, id: string): string => { const service = state.services.get(id); if (service === undefined) return `${id} is unknown`; for (const prerequisite of state.graph.prerequisiteClosure.get(id) ?? []) { @@ -427,7 +420,7 @@ const failAllWaiters = ( /** * Fails every explicit operation's wait on a transitive dependent of a prerequisite that just * failed or lost readiness: such a wait has no budget, so it must end with the prerequisite's - * outcome. Traffic waiters keep their demand and stay bounded by their own budget. + * outcome. Traffic waiters keep their demand and stay bounded by the orchestrator's budget. */ const failExplicitDependentWaiters = ( state: LifecycleState, @@ -501,8 +494,6 @@ const openWaiter = ( waiterId: number, requireReady: boolean, traffic: boolean, - now: number, - openedAt = now, ): readonly [LifecycleState, ReadonlyArray] => { const service = state.services.get(id); const spec = state.graph.services.get(id); @@ -513,9 +504,7 @@ const openWaiter = ( const eligibleNow = requireReady ? admissionReady(state, id) : sessionReady(state, id); if (eligibleNow) return [ - traffic - ? setService(state, id, (s) => ({ ...s, leases: s.leases + 1, lastActivity: now })) - : state, + traffic ? setService(state, id, (s) => ({ ...s, leases: s.leases + 1 })) : state, [LifecycleCommand.AdmitConnection({ id, waiterId })], ]; if (isBreakerOpen(service.breaker)) @@ -523,13 +512,14 @@ const openWaiter = ( // The cap bounds sockets held by traffic; explicit operations are few and already bounded. const trafficWaiters = [...service.waiters.values()].filter(isLeaseWaiter).length; if (traffic && trafficWaiters >= waiterCap) return fail(`${id} has too many waiters`, undefined); - const deadline = traffic ? openedAt + waiterBudgetMillis : undefined; - const commands: Array = - deadline === undefined ? [] : [LifecycleCommand.ArmWaiterTimeout({ id, waiterId, deadline })]; + const commands: Array = []; const queued = setService(state, id, (s) => ({ ...s, - lastActivity: traffic ? now : s.lastActivity, - waiters: new Map(s.waiters).set(waiterId, { id: waiterId, deadline, requireReady }), + waiters: new Map(s.waiters).set(waiterId, { + id: waiterId, + kind: traffic ? "traffic" : "explicit", + requireReady, + }), })); return [retryBlockedCleanup(queued, id, commands), commands]; }; @@ -660,24 +650,12 @@ const applyEvent = ( switch (event._tag) { case "ConnectionOpened": - return openWaiter( - state, - event.id, - event.waiterId, - event.requireReady, - true, - now, - event.openedAt, - ); + return openWaiter(state, event.id, event.waiterId, event.requireReady, true); case "ReadinessAwaited": - return openWaiter(state, event.id, event.waiterId, event.requireReady, false, now); + return openWaiter(state, event.id, event.waiterId, event.requireReady, false); case "ConnectionClosed": { return [ - setService(state, event.id, (s) => ({ - ...s, - leases: Math.max(0, s.leases - 1), - lastActivity: now, - })), + setService(state, event.id, (s) => ({ ...s, leases: Math.max(0, s.leases - 1) })), commands, ]; } @@ -1013,28 +991,6 @@ const applyEvent = ( commands, ]; } - case "WaiterExpired": { - const { id, waiterId } = event; - const service = state.services.get(id); - const waiter = service?.waiters.get(waiterId); - if (service === undefined || waiter === undefined) break; - commands.push( - LifecycleCommand.FailConnection({ - id, - waiterId, - message: `${id} wake budget exceeded while waiting for ${blockingStage(state, id)}`, - cause: undefined, - }), - ); - return [ - setService(state, id, (s) => { - const waiters = new Map(s.waiters); - waiters.delete(waiterId); - return { ...s, waiters }; - }), - commands, - ]; - } case "WaiterCancelled": { const { id, waiterId } = event; return [ @@ -1078,7 +1034,7 @@ const applyEvent = ( const generationCounters = new Map(state.generationCounters); for (const spec of graph.services.values()) { const existing = state.services.get(spec.id); - services.set(spec.id, existing ?? { ...initialService(spec, now), intent: "stopped" }); + services.set(spec.id, existing ?? { ...initialService(spec), intent: "stopped" }); if (!generationCounters.has(spec.id)) generationCounters.set(spec.id, 1); } return [{ graph, services, generationCounters }, commands]; @@ -1092,15 +1048,12 @@ const applyEvent = ( * Cascades launches, idle arming and waiter resolution across the graph after one event. A * single topological pass suffices: a launch started here only reaches `Running` on a later * event, so a dependent several hops away naturally waits for its own future turn. Every - * condition here is a function of state alone, never of `now` against a stored deadline, so - * replaying this pass for a stale or unrelated event can never surface a new launch or admission - * on its own; only the dedicated time-driven events (`CooldownElapsed`, `IdleElapsed`, - * `WaiterExpired`) do that, and the one exception — expiring an already-due waiter instead of - * admitting it — only fires from within this same pass's own admission step. + * condition here is a function of state alone, so replaying this pass for a stale or unrelated + * event can never surface a new launch or admission on its own; only the dedicated time-driven + * events (`CooldownElapsed`, `IdleElapsed`) do that. */ const settle = ( state: LifecycleState, - now: number, commands: ReadonlyArray, ): readonly [LifecycleState, ReadonlyArray] => { let next = state; @@ -1163,38 +1116,19 @@ const settle = ( if (prerequisitesSatisfied(next, id) && service.waiters.size > 0) { const ready = service.phase.ready; const admitted: Array = []; - const expired: Array = []; - for (const waiter of service.waiters.values()) { - if (waiter.requireReady && !ready) continue; - if (waiter.deadline !== undefined && now >= waiter.deadline) expired.push(waiter.id); - else admitted.push(waiter.id); - } - if (admitted.length > 0 || expired.length > 0) { + for (const waiter of service.waiters.values()) + if (!waiter.requireReady || ready) admitted.push(waiter.id); + if (admitted.length > 0) { const leased = admitted.filter((waiterId) => { const waiter = service.waiters.get(waiterId); return waiter !== undefined && isLeaseWaiter(waiter); }).length; for (const waiterId of admitted) emitted.push(LifecycleCommand.AdmitConnection({ id, waiterId })); - for (const waiterId of expired) - emitted.push( - LifecycleCommand.FailConnection({ - id, - waiterId, - message: `${id} wake budget exceeded while waiting for ${blockingStage(next, id)}`, - cause: undefined, - }), - ); next = setService(next, id, (s) => { const waiters = new Map(s.waiters); for (const waiterId of admitted) waiters.delete(waiterId); - for (const waiterId of expired) waiters.delete(waiterId); - return { - ...s, - leases: s.leases + leased, - waiters, - lastActivity: leased > 0 ? now : s.lastActivity, - }; + return { ...s, leases: s.leases + leased, waiters }; }); } } @@ -1237,17 +1171,5 @@ export const reduce = ( now: number, ): readonly [LifecycleState, ReadonlyArray] => { const [applied, directCommands] = applyEvent(state, event, now); - return settle(applied, now, directCommands); -}; - -/** A storage operation is admitted only while a service is stopped, unarmed and not being destroyed. */ -export const canRunStorage = (state: LifecycleState, id: string): boolean => { - const service = state.services.get(id); - return ( - service !== undefined && - service.intent === "stopped" && - service.phase._tag === "Stopped" && - !service.storageReserved && - service.destroy === undefined - ); + return settle(applied, directCommands); }; diff --git a/packages/stack/src/Lifecycle.unit.test.ts b/packages/stack/src/Lifecycle.unit.test.ts index 7247bc8acf..ec204e50e8 100644 --- a/packages/stack/src/Lifecycle.unit.test.ts +++ b/packages/stack/src/Lifecycle.unit.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "@effect/vitest"; import { - canRunStorage, + blockingStage, initialState, LifecycleEvent, makeGraph, @@ -39,6 +39,12 @@ const run = ( const tagsOf = (commands: ReadonlyArray | undefined) => (commands ?? []).map((command) => command._tag); +/** Whether a storage reservation requested now would be accepted rather than rejected. */ +const storageAdmitted = (state: LifecycleState, id: string): boolean => + !tagsOf( + run(state, [{ event: LifecycleEvent.StorageReserved({ id }), now: 0 }]).commandsByStep[0], + ).includes("RequestRejected"); + /** Narrows a service's current phase to `Starting` and returns its generation, or throws. */ const startingGeneration = (state: LifecycleState, id: string): number => { const phase = state.services.get(id)?.phase; @@ -76,13 +82,13 @@ const commandTagged = ( describe("Lifecycle reducer", () => { it("wakes a lazy service on first traffic, admits the waiter once ready, and sleeps after idle", () => { const graph = makeGraph([lazy("api")]); - const { commandsByStep, state } = run(initialState(graph, 0), [ + const { commandsByStep, state } = run(initialState(graph), [ { event: open("api", 1), now: 0 }, { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 20 }, ]); - expect(tagsOf(commandsByStep[0])).toEqual(["ArmWaiterTimeout", "Launch"]); + expect(tagsOf(commandsByStep[0])).toEqual(["Launch"]); expect(tagsOf(commandsByStep[1])).toEqual(["AdmitConnection"]); expect(tagsOf(commandsByStep[2])).toEqual(["ArmIdleTimer"]); const armed = at(at(commandsByStep, 2), 0) as Extract; @@ -96,7 +102,7 @@ describe("Lifecycle reducer", () => { it("ignores a launch outcome, an idle elapse and an exit tagged with a stale generation", () => { const graph = makeGraph([lazy("api")]); - const { state, commandsByStep } = run(initialState(graph, 0), [ + const { state, commandsByStep } = run(initialState(graph), [ { event: open("api", 1), now: 0 }, { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 99 }), now: 10 }, { event: LifecycleEvent.IdleElapsed({ id: "api", generation: 1, epoch: 99 }), now: 20 }, @@ -123,7 +129,7 @@ describe("Lifecycle reducer", () => { it("ignores an idle elapse tagged with a stale epoch even while a newer timer is armed with zero demand", () => { const graph = makeGraph([lazy("api")]); - const { state } = run(initialState(graph, 0), [ + const { state } = run(initialState(graph), [ { event: open("api", 1), now: 0 }, { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 20 }, // arms epoch 1 @@ -146,7 +152,7 @@ describe("Lifecycle reducer", () => { it("keeps a service running when traffic is admitted before the idle commit", () => { const graph = makeGraph([lazy("api")]); - const { state, commandsByStep } = run(initialState(graph, 0), [ + const { state, commandsByStep } = run(initialState(graph), [ { event: open("api", 1), now: 0 }, { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 20 }, @@ -163,7 +169,7 @@ describe("Lifecycle reducer", () => { it("holds traffic arriving after a committed sleep for the confirmed stop and the next generation, without failing it", () => { const graph = makeGraph([lazy("api")]); - const { state, commandsByStep } = run(initialState(graph, 0), [ + const { state, commandsByStep } = run(initialState(graph), [ { event: open("api", 1), now: 0 }, { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 20 }, @@ -183,7 +189,7 @@ describe("Lifecycle reducer", () => { ]); expect(tagsOf(commandsByStep[3])).toEqual(["Stop"]); - expect(tagsOf(commandsByStep[4])).toEqual(["ArmWaiterTimeout"]); + expect(tagsOf(commandsByStep[4])).toEqual([]); expect(tagsOf(commandsByStep[5])).toEqual(["Launch"]); expect((at(at(commandsByStep, 5), 0) as Extract).generation).toBe( 2, @@ -203,7 +209,7 @@ describe("Lifecycle reducer", () => { lazy("functions", { prerequisites: ["database"] }), lazy("studio", { prerequisites: ["pg-meta", "functions"] }), ]); - let state = initialState(graph, 0); + let state = initialState(graph); let commandsByStep: ReadonlyArray>; ({ state } = run(state, [{ event: open("studio", 1), now: 0 }])); @@ -326,7 +332,7 @@ describe("Lifecycle reducer", () => { lazy("b", { prerequisites: ["a"] }), lazy("c", { prerequisites: ["b"] }), ]); - let state = initialState(graph, 0); + let state = initialState(graph); let commandsByStep: ReadonlyArray>; ({ state } = run(state, [{ event: open("c", 1), now: 0 }])); @@ -352,21 +358,16 @@ describe("Lifecycle reducer", () => { expect(state.services.get("c")?.phase).toEqual({ _tag: "Running", generation: 1, ready: true }); const blocked = run(state, [{ event: open("c", 2), now: 50 }]); - expect(tagsOf(blocked.commandsByStep[0])).toEqual(["ArmWaiterTimeout", "Reprobe"]); + expect(tagsOf(blocked.commandsByStep[0])).toEqual(["Reprobe"]); expect(commandTagged(at(blocked.commandsByStep, 0), "Reprobe").id).toBe("a"); expect(blocked.state.services.get("c")?.waiters.size).toBe(1); - - const expired = run(blocked.state, [ - { event: LifecycleEvent.WaiterExpired({ id: "c", waiterId: 2 }), now: 50 + 120_000 + 1 }, - ]); - const failure = commandTagged(at(expired.commandsByStep, 0), "FailConnection"); - expect(failure.message).toContain("prerequisite a"); + expect(blockingStage(blocked.state, "c")).toBe("prerequisite a"); }); describe("explicit start and restart demand", () => { it("launches an idle lazy service on its own explicit start, with no connection involved", () => { const graph = makeGraph([lazy("api")]); - const { commandsByStep } = run(initialState(graph, 0), [ + const { commandsByStep } = run(initialState(graph), [ { event: LifecycleEvent.StartRequested({ id: "api" }), now: 0 }, ]); expect(tagsOf(commandsByStep[0])).toEqual(["Launch"]); @@ -374,7 +375,7 @@ describe("Lifecycle reducer", () => { it("restarts after an explicit stop, even though restart alone would otherwise leave intent stopped", () => { const graph = makeGraph([lazy("api")]); - let state = initialState(graph, 0); + let state = initialState(graph); ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "api" }), now: 0 }])); const restarted = run(state, [ @@ -386,7 +387,7 @@ describe("Lifecycle reducer", () => { it("propagates an explicit restart's demand to a fully stopped prerequisite", () => { const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); - const { commandsByStep } = run(initialState(graph, 0), [ + const { commandsByStep } = run(initialState(graph), [ { event: LifecycleEvent.RestartRequested({ id: "b" }), now: 0 }, ]); expect(tagsOf(commandsByStep[0])).toEqual(["Launch"]); @@ -397,7 +398,7 @@ describe("Lifecycle reducer", () => { describe("circuit breaker", () => { it("opens after 3 consecutive failures, fails fast naming the cause, and recovers once after the cooldown for several waiters at once", () => { const graph = makeGraph([lazy("flaky")]); - let state = initialState(graph, 0); + let state = initialState(graph); let commandsByStep: ReadonlyArray>; const failAttempt = (waiterId: number, now: number) => { @@ -440,13 +441,13 @@ describe("Lifecycle reducer", () => { // After the cooldown elapses, several waiters share exactly one recovery attempt. ({ state, commandsByStep } = run(state, [{ event: open("flaky", 5), now: 201 + 30_000 }])); - expect(tagsOf(commandsByStep[0])).toEqual(["ArmWaiterTimeout", "Launch"]); + expect(tagsOf(commandsByStep[0])).toEqual(["Launch"]); const recovery = commandTagged(at(commandsByStep, 0), "Launch"); ({ state, commandsByStep } = run(state, [ { event: open("flaky", 6), now: 201 + 30_000 + 1 }, ])); - expect(tagsOf(commandsByStep[0])).toEqual(["ArmWaiterTimeout"]); + expect(tagsOf(commandsByStep[0])).toEqual([]); ({ state, commandsByStep } = run(state, [ { @@ -459,7 +460,7 @@ describe("Lifecycle reducer", () => { it("reopens for the same fixed cooldown on a further failure, and an explicit start resets it", () => { const graph = makeGraph([lazy("flaky")]); - let state = initialState(graph, 0); + let state = initialState(graph); const fail = (waiterId: number, now: number) => { ({ state } = run(state, [{ event: open("flaky", waiterId), now }])); @@ -500,12 +501,12 @@ describe("Lifecycle reducer", () => { it("schedules a cooldown timer when the breaker opens and only the dedicated event retries, even with demand retained the whole time", () => { const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); - let state = initialState(graph, 0); + let state = initialState(graph); let commandsByStep: ReadonlyArray>; // b's own waiter on a never resolves, so it keeps retaining demand on a across every retry. ({ state, commandsByStep } = run(state, [{ event: open("b", 1), now: 0 }])); - expect(tagsOf(commandsByStep[0])).toEqual(["ArmWaiterTimeout", "Launch"]); + expect(tagsOf(commandsByStep[0])).toEqual(["Launch"]); const failRetained = (now: number) => { const generation = startingGeneration(state, "a"); @@ -540,7 +541,7 @@ describe("Lifecycle reducer", () => { it("leaves a stale launch outcome a complete no-op even with demand retained and the cooldown elapsed on the wall clock", () => { const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); - let state = initialState(graph, 0); + let state = initialState(graph); // b's own waiter on a is never resolved, so demand on a is retained through every failure: // under the old time-based breaker check, this is exactly the state a stale event could @@ -568,9 +569,9 @@ describe("Lifecycle reducer", () => { }); }); - it("caps waiters at 256, lets an expiry fail only that waiter while the shared launch continues, and frees its slot for cancellation", () => { + it("caps waiters at 256 and frees a slot when a waiter is cancelled", () => { const graph = makeGraph([lazy("api")]); - let state = initialState(graph, 0); + let state = initialState(graph); let commandsByStep: ReadonlyArray>; const opens = Array.from({ length: 256 }, (_, index) => ({ @@ -585,43 +586,22 @@ describe("Lifecycle reducer", () => { expect(tagsOf(commandsByStep[0])).toEqual(["FailConnection"]); expect(state.services.get("api")?.waiters.size).toBe(256); - ({ state, commandsByStep } = run(state, [ - { event: LifecycleEvent.WaiterExpired({ id: "api", waiterId: 5 }), now: 120_001 }, - ])); - expect(tagsOf(commandsByStep[0])).toEqual(["FailConnection"]); - const expired = commandTagged(at(commandsByStep, 0), "FailConnection"); - expect(expired.message).toContain("api to finish preparing"); - expect(state.services.get("api")?.waiters.size).toBe(255); - // The shared launch itself is untouched by a waiter's own expiry. - expect(state.services.get("api")?.phase._tag).toBe("Starting"); - ({ state, commandsByStep } = run(state, [ { event: LifecycleEvent.WaiterCancelled({ id: "api", waiterId: 6 }), now: 2 }, ])); expect(commandsByStep[0]).toEqual([]); - expect(state.services.get("api")?.waiters.size).toBe(254); - - ({ state, commandsByStep } = run(state, [{ event: open("api", 300), now: 3 }])); - expect(tagsOf(commandsByStep[0])).toEqual(["ArmWaiterTimeout"]); expect(state.services.get("api")?.waiters.size).toBe(255); - }); - - it("expires a waiter instead of admitting it when its deadline already passed by the time the launch succeeds", () => { - const graph = makeGraph([lazy("api", { idleMillis: undefined })]); - let state = initialState(graph, 0); - ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); - const generation = startingGeneration(state, "api"); + // The shared launch itself is untouched by a waiter's cancellation. + expect(state.services.get("api")?.phase._tag).toBe("Starting"); - const result = run(state, [ - { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation }), now: 120_000 }, - ]); - expect(tagsOf(result.commandsByStep[0])).toEqual(["FailConnection"]); - expect(result.state.services.get("api")?.leases).toBe(0); + ({ state, commandsByStep } = run(state, [{ event: open("api", 300), now: 3 }])); + expect(tagsOf(commandsByStep[0])).toEqual([]); + expect(state.services.get("api")?.waiters.size).toBe(256); }); it("admits a non-ready-requiring acquisition once a live session exists, bypassing target readiness", () => { const graph = makeGraph([lazy("functions")]); - let state = initialState(graph, 0); + let state = initialState(graph); ({ state } = run(state, [{ event: open("functions", 1), now: 0 }])); const generation = startingGeneration(state, "functions"); @@ -643,7 +623,7 @@ describe("Lifecycle reducer", () => { }); const readyRequiring = run(state, [{ event: open("functions", 2), now: 20 }]); - expect(tagsOf(readyRequiring.commandsByStep[0])).toEqual(["ArmWaiterTimeout", "Reprobe"]); + expect(tagsOf(readyRequiring.commandsByStep[0])).toEqual(["Reprobe"]); const inspector = run(state, [ { @@ -660,7 +640,7 @@ describe("Lifecycle reducer", () => { it("treats a readiness loss as non-terminal, re-probing once for the next waiter, unlike an exit which counts toward the breaker", () => { const graph = makeGraph([lazy("api")]); - const { state: ready } = run(initialState(graph, 0), [ + const { state: ready } = run(initialState(graph), [ { event: open("api", 1), now: 0 }, { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, ]); @@ -683,8 +663,8 @@ describe("Lifecycle reducer", () => { { event: open("api", 2), now: 21 }, { event: open("api", 3), now: 22 }, ]); - expect(tagsOf(probeCommands[0])).toEqual(["ArmWaiterTimeout", "Reprobe"]); - expect(tagsOf(probeCommands[1])).toEqual(["ArmWaiterTimeout"]); + expect(tagsOf(probeCommands[0])).toEqual(["Reprobe"]); + expect(tagsOf(probeCommands[1])).toEqual([]); const { state: recovered, commandsByStep: recoveredCommands } = run(probing, [ { event: LifecycleEvent.ReadinessRecovered({ id: "api", generation: 1 }), now: 30 }, @@ -720,7 +700,7 @@ describe("Lifecycle reducer", () => { it("rejects stopping or restarting a prerequisite while a dependent is active", () => { const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); - let state = initialState(graph, 0); + let state = initialState(graph); ({ state } = run(state, [{ event: open("b", 1), now: 0 }])); ({ state } = run(state, [ { event: LifecycleEvent.LaunchSucceeded({ id: "a", generation: 1 }), now: 10 }, @@ -748,24 +728,24 @@ describe("Lifecycle reducer", () => { describe("storage reservation", () => { it("admits storage operations only once a service is stopped with intent stopped", () => { const graph = makeGraph([lazy("database")]); - let state = initialState(graph, 0); - expect(canRunStorage(state, "database")).toBe(false); + let state = initialState(graph); + expect(storageAdmitted(state, "database")).toBe(false); ({ state } = run(state, [ { event: LifecycleEvent.StopRequested({ id: "database" }), now: 0 }, ])); - expect(canRunStorage(state, "database")).toBe(true); + expect(storageAdmitted(state, "database")).toBe(true); ({ state } = run(state, [ { event: LifecycleEvent.StartRequested({ id: "database" }), now: 1 }, ])); ({ state } = run(state, [{ event: open("database", 1), now: 2 }])); - expect(canRunStorage(state, "database")).toBe(false); + expect(storageAdmitted(state, "database")).toBe(false); }); it("rejects a storage reservation unless the service is already stopped", () => { const graph = makeGraph([lazy("database")]); - const { commandsByStep } = run(initialState(graph, 0), [ + const { commandsByStep } = run(initialState(graph), [ { event: LifecycleEvent.StorageReserved({ id: "database" }), now: 0 }, ]); expect(tagsOf(commandsByStep[0])).toEqual(["RequestRejected"]); @@ -773,7 +753,7 @@ describe("Lifecycle reducer", () => { it("blocks a launch while storage is reserved and relaunches once it's released", () => { const graph = makeGraph([lazy("database")]); - let state = initialState(graph, 0); + let state = initialState(graph); ({ state } = run(state, [ { event: LifecycleEvent.StopRequested({ id: "database" }), now: 0 }, ])); @@ -800,7 +780,7 @@ describe("Lifecycle reducer", () => { it("hands a running storage operation's reservation to a destroy requested meanwhile", () => { const graph = makeGraph([lazy("database")]); - let state = initialState(graph, 0); + let state = initialState(graph); ({ state } = run(state, [ { event: LifecycleEvent.StopRequested({ id: "database" }), now: 0 }, { event: LifecycleEvent.StorageReserved({ id: "database" }), now: 1 }, @@ -816,7 +796,7 @@ describe("Lifecycle reducer", () => { ["RequestRejected"], ["RequestRejected"], ]); - expect(canRunStorage(refused.state, "database")).toBe(false); + expect(storageAdmitted(refused.state, "database")).toBe(false); ({ state } = run(refused.state, [ { event: LifecycleEvent.StorageReleased({ id: "database" }), now: 4 }, @@ -829,7 +809,7 @@ describe("Lifecycle reducer", () => { it("reserves storage for a destroy once its running service has stopped", () => { const graph = makeGraph([lazy("database")]); - let state = initialState(graph, 0); + let state = initialState(graph); ({ state } = run(state, [ { event: LifecycleEvent.StartRequested({ id: "database" }), now: 0 }, ])); @@ -861,7 +841,7 @@ describe("Lifecycle reducer", () => { it("makes a service usable again once a destroy releases it", () => { const graph = makeGraph([lazy("database")]); - let state = initialState(graph, 0); + let state = initialState(graph); ({ state } = run(state, [ { event: LifecycleEvent.StopRequested({ id: "database" }), now: 0 }, { event: LifecycleEvent.DestroyRequested({ id: "database" }), now: 1 }, @@ -871,7 +851,7 @@ describe("Lifecycle reducer", () => { ({ state } = run(state, [ { event: LifecycleEvent.StorageReleased({ id: "database" }), now: 2 }, ])); - expect(canRunStorage(state, "database")).toBe(false); + expect(storageAdmitted(state, "database")).toBe(false); ({ state } = run(state, [ { event: LifecycleEvent.DestroyReleased({ id: "database" }), now: 3 }, @@ -880,13 +860,13 @@ describe("Lifecycle reducer", () => { destroy: undefined, storageReserved: false, }); - expect(canRunStorage(state, "database")).toBe(true); + expect(storageAdmitted(state, "database")).toBe(true); }); }); it("rejects new admission for an explicitly stopped service without creating a waiter", () => { const graph = makeGraph([lazy("api")]); - let state = initialState(graph, 0); + let state = initialState(graph); ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "api" }), now: 0 }])); const result = run(state, [{ event: open("api", 1), now: 1 }]); @@ -899,7 +879,7 @@ describe("Lifecycle reducer", () => { lazy("database", { idleMillis: undefined }), { id: "kong", activation: "eager", prerequisites: ["database"] }, ]); - let state = initialState(graph, 0); + let state = initialState(graph); let commandsByStep: ReadonlyArray>; // Kong is eager from composition, so the owner's first settle pass (triggered by any event, @@ -917,7 +897,7 @@ describe("Lifecycle reducer", () => { describe("graph updates", () => { it("preserves a continuing service's state and never reuses a generation after a remove and re-add", () => { - let state = initialState(makeGraph([lazy("api")]), 0); + let state = initialState(makeGraph([lazy("api")])); ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); ({ state } = run(state, [ { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, @@ -956,7 +936,7 @@ describe("Lifecycle reducer", () => { }); it("rejects removing a service that's starting with a pending waiter, atomically", () => { - let state = initialState(makeGraph([lazy("a")]), 0); + let state = initialState(makeGraph([lazy("a")])); ({ state } = run(state, [{ event: open("a", 1), now: 0 }])); expect(state.services.get("a")?.phase._tag).toBe("Starting"); @@ -968,7 +948,7 @@ describe("Lifecycle reducer", () => { }); it("rejects removing a service with a storage reservation, so a graph update can't launch mid-restore", () => { - let state = initialState(makeGraph([lazy("db")]), 0); + let state = initialState(makeGraph([lazy("db")])); ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "db" }), now: 0 }])); ({ state } = run(state, [{ event: LifecycleEvent.StorageReserved({ id: "db" }), now: 1 }])); expect(state.services.get("db")?.storageReserved).toBe(true); @@ -982,7 +962,7 @@ describe("Lifecycle reducer", () => { it("rejects changing an active continuing service's prerequisites, leaving its live generation untouched", () => { const graphV1 = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); - let state = initialState(graphV1, 0); + let state = initialState(graphV1); ({ state } = run(state, [{ event: open("b", 1), now: 0 }])); ({ state } = run(state, [ { event: LifecycleEvent.LaunchSucceeded({ id: "a", generation: 1 }), now: 10 }, @@ -1010,7 +990,7 @@ describe("Lifecycle reducer", () => { describe("stage reporting", () => { it("updates the starting stage so a waiter's budget error names it", () => { - let state = initialState(makeGraph([lazy("api")]), 0); + let state = initialState(makeGraph([lazy("api")])); ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); const generation = startingGeneration(state, "api"); @@ -1026,15 +1006,11 @@ describe("Lifecycle reducer", () => { stage: "launching", }); - const expired = run(state, [ - { event: LifecycleEvent.WaiterExpired({ id: "api", waiterId: 1 }), now: 100 }, - ]); - const failure = commandTagged(at(expired.commandsByStep, 0), "FailConnection"); - expect(failure.message).toContain("api to finish launching"); + expect(blockingStage(state, "api")).toBe("api to finish launching"); }); it("ignores a stage change tagged with a stale generation", () => { - let state = initialState(makeGraph([lazy("api")]), 0); + let state = initialState(makeGraph([lazy("api")])); ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); const result = run(state, [ { @@ -1053,7 +1029,7 @@ describe("Lifecycle reducer", () => { lazy("b", { prerequisites: ["a"] }), lazy("c", { prerequisites: ["b"] }), ]); - let state = initialState(graph, 0); + let state = initialState(graph); ({ state } = run(state, [{ event: open("c", 1), now: 0 }])); expect(state.services.get("a")?.phase._tag).toBe("Starting"); expect(state.services.get("b")?.phase._tag).toBe("Stopped"); @@ -1068,7 +1044,7 @@ describe("Lifecycle reducer", () => { it("still protects a dependent's demand once the prerequisite's own breaker has opened", () => { const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); - let state = initialState(graph, 0); + let state = initialState(graph); ({ state } = run(state, [{ event: open("b", 1), now: 0 }])); ({ state } = run(state, [ { event: LifecycleEvent.LaunchSucceeded({ id: "a", generation: 1 }), now: 10 }, @@ -1109,7 +1085,7 @@ describe("Lifecycle reducer", () => { it("blocks an explicit stop or restart of a prerequisite while a sleeping dependent is still wake-armed", () => { const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); - let state = initialState(graph, 0); + let state = initialState(graph); ({ state } = run(state, [{ event: open("b", 1), now: 0 }])); ({ state } = run(state, [ { event: LifecycleEvent.LaunchSucceeded({ id: "a", generation: 1 }), now: 10 }, @@ -1160,7 +1136,7 @@ describe("Lifecycle reducer", () => { it("re-arms a stopped prerequisite when its dependent is explicitly restarted", () => { const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); - let state = initialState(graph, 0); + let state = initialState(graph); ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "b" }), now: 0 }])); ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "a" }), now: 1 }])); expect(state.services.get("a")?.intent).toBe("stopped"); @@ -1172,7 +1148,7 @@ describe("Lifecycle reducer", () => { }); it("keeps an explicit start's demand through a committed stop so the next generation launches once the exit is confirmed", () => { - let state = initialState(makeGraph([lazy("api")]), 0); + let state = initialState(makeGraph([lazy("api")])); ({ state } = run(state, [{ event: LifecycleEvent.StartRequested({ id: "api" }), now: 0 }])); expect(state.services.get("api")?.phase._tag).toBe("Starting"); ({ state } = run(state, [ @@ -1202,7 +1178,7 @@ describe("Lifecycle reducer", () => { }); it("admits a cold inspector acquisition before the first readiness check while ordinary traffic still waits", () => { - let state = initialState(makeGraph([lazy("functions")]), 0); + let state = initialState(makeGraph([lazy("functions")])); ({ state } = run(state, [{ event: open("functions", 1), now: 0 }])); const generation = startingGeneration(state, "functions"); expect(state.services.get("functions")?.waiters.size).toBe(1); @@ -1217,7 +1193,7 @@ describe("Lifecycle reducer", () => { now: 1, }, ]); - expect(tagsOf(inspectorQueued.commandsByStep[0])).toEqual(["ArmWaiterTimeout"]); + expect(tagsOf(inspectorQueued.commandsByStep[0])).toEqual([]); state = inspectorQueued.state; const result = run(state, [ @@ -1235,7 +1211,7 @@ describe("Lifecycle reducer", () => { }); it("re-arms the idle timer once demand returns and readiness later recovers, instead of staying armed forever", () => { - let state = initialState(makeGraph([lazy("api")]), 0); + let state = initialState(makeGraph([lazy("api")])); ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); ({ state } = run(state, [ { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, @@ -1257,17 +1233,17 @@ describe("Lifecycle reducer", () => { expect(staleElapse.commandsByStep[0]).toEqual([]); ({ state } = run(state, [ - { event: LifecycleEvent.WaiterExpired({ id: "api", waiterId: 2 }), now: 120_040 }, + { event: LifecycleEvent.WaiterCancelled({ id: "api", waiterId: 2 }), now: 60 }, ])); ({ state } = run(state, [ - { event: LifecycleEvent.ReadinessRecovered({ id: "api", generation: 1 }), now: 120_041 }, + { event: LifecycleEvent.ReadinessRecovered({ id: "api", generation: 1 }), now: 61 }, ])); expect(state.services.get("api")?.idleArmedEpoch).toBe(2); }); it("fails only the waiters awaiting a failed readiness check, keeping the session and inspector waiters", () => { const graph = makeGraph([lazy("database"), lazy("api", { prerequisites: ["database"] })]); - let state = initialState(graph, 0); + let state = initialState(graph); ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); ({ state } = run(state, [ { event: LifecycleEvent.LaunchSucceeded({ id: "database", generation: 1 }), now: 1 }, @@ -1290,7 +1266,7 @@ describe("Lifecycle reducer", () => { }); it("gives an explicit readiness wait no budget and no lease, while it still wakes the service", () => { - let state = initialState(makeGraph([lazy("api")]), 0); + let state = initialState(makeGraph([lazy("api")])); let commandsByStep: ReadonlyArray>; ({ state, commandsByStep } = run(state, [ { @@ -1308,7 +1284,7 @@ describe("Lifecycle reducer", () => { }); it("adds services unarmed through a graph update until an arm or start request", () => { - let state = initialState(makeGraph([]), 0); + let state = initialState(makeGraph([])); let commandsByStep: ReadonlyArray>; const graph = makeGraph([ { id: "database", activation: "eager", prerequisites: [] }, @@ -1336,7 +1312,7 @@ describe("Lifecycle reducer", () => { it("lets an eager service's standing demand lapse once it fails, until traffic re-wakes it", () => { const graph = makeGraph([{ id: "database", activation: "eager", prerequisites: [] }]); - let state = initialState(graph, 0); + let state = initialState(graph); let commandsByStep: ReadonlyArray>; ({ state, commandsByStep } = run(state, [ { event: LifecycleEvent.StartRequested({ id: "database" }), now: 0 }, @@ -1354,7 +1330,7 @@ describe("Lifecycle reducer", () => { it("ends an explicit wait on a dependent with its prerequisite's failure, while traffic keeps waiting", () => { const graph = makeGraph([lazy("database"), lazy("api", { prerequisites: ["database"] })]); - let state = initialState(graph, 0); + let state = initialState(graph); ({ state } = run(state, [ { event: open("api", 1), now: 0 }, { @@ -1384,7 +1360,7 @@ describe("Lifecycle reducer", () => { lazy("analytics"), lazy("studio", { prerequisites: ["analytics"] }), ]); - let state = initialState(graph, 0); + let state = initialState(graph); ({ state } = run(state, [ { event: open("studio", 1), now: 0 }, { event: LifecycleEvent.SessionAvailable({ id: "analytics", generation: 1 }), now: 1 }, @@ -1412,7 +1388,7 @@ describe("Lifecycle reducer", () => { generation: 1, delayMillis: 0, }); - expect(tagsOf(commandsByStep[1])).toEqual(["ArmWaiterTimeout"]); + expect(tagsOf(commandsByStep[1])).toEqual([]); expect(commandTagged(at(commandsByStep, 2), "Reprobe").delayMillis).toBe(1_000); expect(commandTagged(at(commandsByStep, 3), "Reprobe").delayMillis).toBe(1_000); expect(commandTagged(at(commandsByStep, 4), "Launch").id).toBe("studio"); @@ -1424,7 +1400,7 @@ describe("Lifecycle reducer", () => { let commandsByStep: ReadonlyArray>; ({ state } = run(state, [lost(2)])); ({ state, commandsByStep } = run(state, [ - { event: LifecycleEvent.WaiterExpired({ id: "studio", waiterId: 1 }), now: 3 }, + { event: LifecycleEvent.WaiterCancelled({ id: "studio", waiterId: 1 }), now: 3 }, lost(4), ])); expect(tagsOf(commandsByStep[1])).toEqual([]); @@ -1448,7 +1424,7 @@ describe("Lifecycle reducer", () => { describe("failed cleanup", () => { const runningApi = () => { - let state = initialState(makeGraph([lazy("api")]), 0); + let state = initialState(makeGraph([lazy("api")])); ({ state } = run(state, [ { event: LifecycleEvent.StartRequested({ id: "api" }), now: 0 }, { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 1 }, @@ -1469,7 +1445,7 @@ describe("Lifecycle reducer", () => { expect(tagsOf(commandsByStep[0])).toEqual(["Stop"]); expect(commandsByStep[1]).toEqual([]); expect(state.services.get("api")?.phase).toEqual({ _tag: "Stopping", generation: 1 }); - expect(canRunStorage(state, "api")).toBe(false); + expect(storageAdmitted(state, "api")).toBe(false); ({ state, commandsByStep } = run(state, [ { event: LifecycleEvent.StorageReserved({ id: "api" }), now: 4 }, @@ -1491,7 +1467,7 @@ describe("Lifecycle reducer", () => { now: 7, }, ])); - expect(canRunStorage(state, "api")).toBe(true); + expect(storageAdmitted(state, "api")).toBe(true); }); it("counts a crash whose cleanup failed once, not again when the retried cleanup confirms it", () => { @@ -1550,12 +1526,12 @@ describe("Lifecycle reducer", () => { expect(state.services.get("api")?.relaunchForced).toBe(false); ({ commandsByStep } = run(state, [{ event: open("api", 2), now: 4 }])); - expect(tagsOf(commandsByStep[0])).toEqual(["ArmWaiterTimeout", "Stop"]); + expect(tagsOf(commandsByStep[0])).toEqual(["Stop"]); }); }); - it("caps only traffic waiters and dates a waiter's budget from the client's arrival", () => { - let state = initialState(makeGraph([lazy("api")]), 0); + it("caps only traffic waiters, keeps explicit waits out of the count and frees a slot on cancellation", () => { + let state = initialState(makeGraph([lazy("api")])); let commandsByStep: ReadonlyArray>; ({ state } = run( state, @@ -1567,35 +1543,18 @@ describe("Lifecycle reducer", () => { event: LifecycleEvent.ReadinessAwaited({ id: "api", waiterId: 1_001, requireReady: true }), now: 0, }, - { - event: LifecycleEvent.ConnectionOpened({ - id: "api", - waiterId: 1_002, - requireReady: true, - openedAt: 0, - }), - now: 40, - }, ])); expect(commandTagged(at(commandsByStep, 0), "FailConnection").message).toBe( "api has too many waiters", ); expect(commandsByStep[1]).toEqual([]); - expect(state.services.get("api")?.waiters.has(1_001)).toBe(true); - expect(tagsOf(commandsByStep[2])).toEqual(["FailConnection"]); + expect(state.services.get("api")?.waiters.get(1_001)?.kind).toBe("explicit"); ({ state, commandsByStep } = run(state, [ { event: LifecycleEvent.WaiterCancelled({ id: "api", waiterId: 0 }), now: 50 }, - { - event: LifecycleEvent.ConnectionOpened({ - id: "api", - waiterId: 1_003, - requireReady: true, - openedAt: 20, - }), - now: 50, - }, + { event: open("api", 1_003), now: 50 }, ])); - expect(commandTagged(at(commandsByStep, 1), "ArmWaiterTimeout").deadline).toBe(120_020); + expect(commandsByStep[1]).toEqual([]); + expect(state.services.get("api")?.waiters.get(1_003)?.kind).toBe("traffic"); }); }); diff --git a/packages/stack/src/Orchestrator.integration.test.ts b/packages/stack/src/Orchestrator.integration.test.ts index 7d68eecdf7..067f41b0b8 100644 --- a/packages/stack/src/Orchestrator.integration.test.ts +++ b/packages/stack/src/Orchestrator.integration.test.ts @@ -1057,6 +1057,62 @@ describe("wake and idle sleep", () => { ).pipe(Effect.provide(TestClock.layer())), ); + describe("a traffic waiter that leaves before its launch finishes", () => { + // A leaked lease or waiter would keep demand on the service and stop it from ever idling. + const idlesAfterWaiterLeaves = ( + leave: ( + waiter: Fiber.Fiber>, + ) => Effect.Effect, + ) => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const launching = yield* Deferred.make(); + const proceed = yield* Deferred.make(); + const api = yield* makeInstance(orchestrator, "api", { + launch: Deferred.succeed(launching, undefined).pipe( + Effect.andThen(Deferred.await(proceed)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy", idleMillis: 1000 }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const waiter = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.exit, + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.await(launching); + yield* leave(waiter); + + yield* Deferred.succeed(proceed, undefined); + yield* api.ready; + yield* TestClock.adjust("1 second"); + yield* stopped(api); + expect(yield* Ref.get(api.starts)).toHaveLength(1); + }), + ).pipe(Effect.provide(TestClock.layer())); + + it.live( + "releases its claim when its wake budget expires, so the service idles afterwards", + () => + idlesAfterWaiterLeaves((waiter) => + TestClock.adjust("120 seconds").pipe( + Effect.andThen(Fiber.join(waiter)), + Effect.tap((exit) => + Effect.sync(() => expect(errorOf(exit)?.message).toContain("wake budget exceeded")), + ), + Effect.asVoid, + ), + ), + ); + + it.live("releases its claim when the client cancels, so the service idles afterwards", () => + idlesAfterWaiterLeaves((waiter) => Fiber.interrupt(waiter)), + ); + }); + it.live("releases only a cancelled waiter and keeps the shared launch for the others", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/Orchestrator.ts b/packages/stack/src/Orchestrator.ts index c95939a7ac..33756fc40b 100644 --- a/packages/stack/src/Orchestrator.ts +++ b/packages/stack/src/Orchestrator.ts @@ -15,11 +15,11 @@ import { SubscriptionRef, } from "effect"; import { + blockingStage, initialState, LifecycleEvent, makeGraph, reduce, - waiterBudgetMillis, type LifecycleCommand, type LifecycleGraph, type LifecycleState, @@ -134,6 +134,9 @@ export const CompositionConfig = Schema.Struct({ ), }); +/** The wake budget a traffic waiter gets, counted from its arrival. */ +const waiterBudgetMillis = 120_000; + /** One service's lifecycle phase and health from the reducer, joined with its execution facts. */ export interface Status extends ServiceObservation { readonly lifecycle: "stopped" | "starting" | "running" | "stopping"; @@ -324,6 +327,16 @@ const lifecycleOf = (state: LifecycleState, id: string): Status["lifecycle"] => } }; +const wakeEnabledOf = (state: LifecycleState, id: string): boolean => { + const intent = state.services.get(id)?.intent; + return intent !== undefined && intent !== "stopped"; +}; + +/** Whether a service is stopped and nothing can wake it, which graph, data and credential changes require. */ +export const isStoppedAndWakeDisabled = ( + status: Pick, +): boolean => status.lifecycle === "stopped" && !status.wakeEnabled; + const statusOf = ( state: LifecycleState, id: string, @@ -353,7 +366,7 @@ const statusOf = ( phase === undefined || phase._tag === "Stopped" || phase._tag === "Failed" ? undefined : phase.generation, - wakeEnabled: service !== undefined && service.intent !== "stopped", + wakeEnabled: wakeEnabledOf(state, id), destroyPending: service?.destroy !== undefined, }; }; @@ -391,9 +404,7 @@ export const make = Effect.fn("Orchestrator.make")(function* Effect.void); // The one lifecycle gate: every reducer step for this owner's graph is applied under it. const gate = yield* Semaphore.make(1); - const lifecycle = yield* SubscriptionRef.make( - initialState(makeGraph([]), yield* Clock.currentTimeMillis), - ); + const lifecycle = yield* SubscriptionRef.make(initialState(makeGraph([]))); const registry = yield* Ref.make>(new Map()); const composition = yield* Ref.make({ members: [], dependencies: [] }); const waiters = yield* Ref.make>(new Map()); @@ -603,7 +614,6 @@ export const make = Effect.fn("Orchestrator.make")(function* ) { + const state = yield* SubscriptionRef.get(lifecycle); + const queued = yield* Ref.get(reached); + return yield* new ServiceError({ + operation: queued && state.services.get(id)?.phase._tag === "Running" ? "readiness" : "wake", + message: `${id} wake budget exceeded while waiting for ${queued ? blockingStage(state, id) : "admission"}`, + }); + }); + interface Admission { /** Explicit requests dispatched atomically before the waiters; a rejection fails the call. */ readonly requests?: ReadonlyArray>; readonly waits: ReadonlyArray; - /** Traffic waits carry a lease and a wake budget; explicit waits carry neither. */ + /** Traffic waits carry a lease and the wake budget; explicit waits carry neither. */ readonly traffic: boolean; /** Checks the owner's admission guard under the same gate step as the requests. */ readonly operation?: AdmittedOperation; @@ -631,12 +651,11 @@ export const make = Effect.fn("Orchestrator.make")(function* Effect.gen(function* () { @@ -656,25 +675,21 @@ export const make = Effect.fn("Orchestrator.make")(function* } >, - openedAt: number, - bounded: Wait | undefined, restore: (effect: Effect.Effect) => Effect.Effect, ) { const { requests = [], traffic } = admission; @@ -699,63 +714,24 @@ export const make = Effect.fn("Orchestrator.make")(function* traffic - ? LifecycleEvent.ConnectionOpened({ id, waiterId, requireReady, openedAt }) + ? LifecycleEvent.ConnectionOpened({ id, waiterId, requireReady }) : LifecycleEvent.ReadinessAwaited({ id, waiterId, requireReady }), ), ]), ), ), ); - const dispatched = yield* restore( - bounded === undefined - ? guarded - : guarded.pipe( - Effect.timeoutOrElse({ - duration: `${waiterBudgetMillis} millis`, - orElse: () => - Effect.fail( - new ServiceError({ - operation: "wake", - message: `${bounded.id} wake budget exceeded while waiting for admission`, - }), - ), - }), - ), - ).pipe(Effect.exit); - if (Exit.isFailure(dispatched)) { - if (yield* Ref.get(reached)) yield* withdraw; - return yield* Effect.failCause(dispatched.cause); - } - const applied = dispatched.value; - for (const request of requests) { - const rejected = rejectionFor(applied.commands, request.id); - if (rejected === undefined) continue; - yield* withdraw; - return yield* rejected; - } - const awaitOne = ({ id, waiterId, deferred }: (typeof entries)[number]) => { - const timeout = applied.commands.find( - (command) => command._tag === "ArmWaiterTimeout" && command.waiterId === waiterId, - ); - if (timeout?._tag !== "ArmWaiterTimeout") return Deferred.await(deferred); - return Clock.currentTimeMillis.pipe( - Effect.flatMap((now) => - Deferred.await(deferred).pipe( - Effect.timeoutOption(`${Math.max(0, timeout.deadline - now)} millis`), - ), - ), - Effect.flatMap((admitted) => - Option.isSome(admitted) - ? Effect.void - : dispatch([LifecycleEvent.WaiterExpired({ id, waiterId })]).pipe( - Effect.andThen(Deferred.await(deferred)), - ), - ), - ); - }; - yield* admission.onApplied?.(applied) ?? Effect.void; - const outcome = yield* restore( - Effect.forEach(entries, awaitOne, { concurrency: "unbounded", discard: true }).pipe( + const admitted = Effect.gen(function* () { + const applied = yield* guarded; + for (const request of requests) { + const rejected = rejectionFor(applied.commands, request.id); + if (rejected !== undefined) return yield* rejected; + } + yield* admission.onApplied?.(applied) ?? Effect.void; + yield* Effect.forEach(entries, ({ deferred }) => Deferred.await(deferred), { + concurrency: "unbounded", + discard: true, + }).pipe( Effect.withSpan("Lifecycle.awaitAdmission", { attributes: { instance_ids: entries.map(({ id }) => id).join(","), @@ -763,11 +739,24 @@ export const make = Effect.fn("Orchestrator.make")(function* budgetExceeded(subject.id, reached), + }), + ); + const outcome = yield* restore(budgeted).pipe(Effect.exit); + if (Exit.isFailure(outcome)) { + if (yield* Ref.get(reached)) yield* withdraw; + } else yield* admission.onAdmitted ?? Effect.void; + return yield* outcome; }); const prerequisitesOf = (state: LifecycleState, id: string): ReadonlyArray => @@ -1241,10 +1230,12 @@ export const make = Effect.fn("Orchestrator.make")(function* orchestrator.status(id).pipe( - Effect.flatMap(({ lifecycle, wakeEnabled }) => - lifecycle === "stopped" && !wakeEnabled + Effect.flatMap((status) => + Orchestrator.isStoppedAndWakeDisabled(status) ? Effect.void : Effect.fail( new CredentialError({ @@ -455,9 +455,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ).pipe(Effect.provideService(Scope.Scope, ownerScope)); // Listeners stay open while the service runs or demand can still wake it. const enabled = orchestrator.status(id).pipe( - Effect.map( - (status) => status.registered && (status.wakeEnabled || status.lifecycle !== "stopped"), - ), + Effect.map((status) => status.registered && !Orchestrator.isStoppedAndWakeDisabled(status)), Effect.orElseSucceed(() => false), ); const configFor = (inputs: Record, candidate: unknown) => diff --git a/packages/stack/src/composition/Supabase.ts b/packages/stack/src/composition/Supabase.ts index 73fa2aee22..3773ecef98 100644 --- a/packages/stack/src/composition/Supabase.ts +++ b/packages/stack/src/composition/Supabase.ts @@ -1,6 +1,6 @@ import { Data, Effect, Exit, Redacted, Schema } from "effect"; import { postgresVersion } from "../Artifacts.ts"; -import { causeMessage, type CompositionConfig } from "../Orchestrator.ts"; +import { causeMessage, type CompositionConfig, isStoppedAndWakeDisabled } from "../Orchestrator.ts"; import type { Observation } from "../Rpc.ts"; import { ServiceCreation, type ServiceCreationInput } from "../services/Catalog.ts"; import type { SavedStack, StackKeysInput } from "../StackNamespace.ts"; @@ -430,7 +430,7 @@ export const makeSupabaseComposition = Effect.fn("Supabase.compose")( operations.status(id).pipe( Effect.mapError(compositionErrorFrom), Effect.flatMap((status) => - status.lifecycle === "stopped" && !status.wakeEnabled + isStoppedAndWakeDisabled(status) ? Effect.void : Effect.fail( compositionError( From 6b69cf267a975dd4f8c2c71d04151cf24089d88c Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 02:05:47 +0200 Subject: [PATCH 070/142] fix(stack): prune destroyed stack directories after the restart window --- packages/stack/ARCHITECTURE.md | 2 +- packages/stack/src/Sweep.integration.test.ts | 48 ++++++++++++++++++++ packages/stack/src/Sweep.ts | 1 + packages/stack/src/namespace/Registry.ts | 43 +++++++++++++++++- 4 files changed, 91 insertions(+), 3 deletions(-) diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 2fd2170142..fa0fcff8a9 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -487,7 +487,7 @@ During Starting, acquire the exclusive stack lease, load the instance definition **Session lifetime.** A session stack is registered by its owner under the lease, so it never exists without a live owner except after that owner dies. Its spawner keeps the owner's stdin pipe open for the life of the creating handle. End of input means the creator is gone, whether it closed the handle or its process died: the owner destroys the stack and exits. Only the creating handle starts a session stack's owner; other handles attach. `create({ startOwner: true })` registers a detached stack through its owner the same way; an owner whose startup fails removes the registration it made, so a failed or interrupted first launch leaves no stack behind. -**Orphan sweep.** Stack-labelled containers and session stacks exist only while their lease is held. After readiness, each owner visits every other stack in its state root in the background, with a bounded time per stack. It skips stacks whose lease is held. For a free lease it takes that lease for the duration of the visit, publishing a sweeper record in `owner.json` so clients wait for the visit instead of mistaking it for a starting owner, removes containers labelled with the stack and its data root, and destroys the stack through the owner's own destroy path when its lifetime is `session`. Filtering on the data-root label keeps other state roots untouched. Creating a stack whose identity belongs to a dead session stack reclaims that stack the same way first. +**Orphan sweep.** Stack-labelled containers and session stacks exist only while their lease is held. After readiness, each owner visits every other stack in its state root in the background, with a bounded time per stack. It skips stacks whose lease is held. For a free lease it takes that lease for the duration of the visit, publishing a sweeper record in `owner.json` so clients wait for the visit instead of mistaking it for a starting owner, removes containers labelled with the stack and its data root, and destroys the stack through the owner's own destroy path when its lifetime is `session`. Filtering on the data-root label keeps other state roots untouched. Creating a stack whose identity belongs to a dead session stack reclaims that stack the same way first. The sweep also removes the empty `/` and `data/` directories a destroy leaves behind, once they are over ten minutes old and hold nothing else. **Abandoned registration.** The orphan sweep above covers a dead owner whose registration survives; the opposite case is a live owner whose registration disappears out from under it (the state root diff --git a/packages/stack/src/Sweep.integration.test.ts b/packages/stack/src/Sweep.integration.test.ts index 3dd45457dc..567bef4a67 100644 --- a/packages/stack/src/Sweep.integration.test.ts +++ b/packages/stack/src/Sweep.integration.test.ts @@ -20,6 +20,7 @@ import { watch as nodeWatch } from "node:fs"; import { launchHost } from "./HostProcess.ts"; import { makeContainerRuntime, resolveEngineTarget } from "./runtime/Container.ts"; import * as StackNamespace from "./StackNamespace.ts"; +import { sweepOrphans } from "./Sweep.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; import { shutdownOwner, watchLeaseRelease } from "../tests/owner.ts"; @@ -235,3 +236,50 @@ it.live.skipIf(process.platform === "win32")( ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 180_000 }, ); + +it.live( + "an orphan sweep removes destroyed stack directories once they are older than the restart window", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-sweep-prune-" }); + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root })), + StackNamespace.Service, + ); + const elevenMinutesAgo = Math.floor((yield* Clock.currentTimeMillis) / 1000) - 11 * 60; + const arrange = Effect.fn("SweepTest.arrange")(function* ( + id: string, + options: { readonly old: boolean; readonly file?: string; readonly dataFile?: string }, + ) { + const directory = path.join(root, id); + yield* fs.makeDirectory(path.join(directory, "data"), { recursive: true }); + if (options.file !== undefined) + yield* fs.writeFileString(path.join(directory, options.file), "x"); + if (options.dataFile !== undefined) + yield* fs.writeFileString(path.join(directory, "data", options.dataFile), "x"); + if (options.old) yield* fs.utimes(directory, elevenMinutesAgo, elevenMinutesAgo); + }); + yield* arrange("old-empty", { old: true }); + yield* arrange("fresh-empty", { old: false }); + yield* arrange("old-with-data", { old: true, dataFile: "PG_VERSION" }); + yield* arrange("old-with-lock", { old: true, file: "owner.lock" }); + yield* arrange("old-kept", { old: true }); + + yield* sweepOrphans({ + state, + stateRoot: root, + cacheRoot: path.join(root, "cache"), + ownerId: "old-kept", + }); + + expect(yield* fs.exists(path.join(root, "old-empty"))).toBe(false); + expect(yield* fs.exists(path.join(root, "fresh-empty", "data"))).toBe(true); + expect(yield* fs.exists(path.join(root, "old-with-data", "data", "PG_VERSION"))).toBe(true); + expect(yield* fs.exists(path.join(root, "old-with-lock", "owner.lock"))).toBe(true); + expect(yield* fs.exists(path.join(root, "old-kept", "data"))).toBe(true); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); diff --git a/packages/stack/src/Sweep.ts b/packages/stack/src/Sweep.ts index 42a2f1d173..4737524447 100644 --- a/packages/stack/src/Sweep.ts +++ b/packages/stack/src/Sweep.ts @@ -82,6 +82,7 @@ export const sweepOrphans = Effect.fn("Sweep.orphans")( ), ); } + yield* options.state.pruneDestroyed(options.ownerId); }, Effect.catchCause((cause) => Effect.logWarning("Orphan sweep failed", cause)), ); diff --git a/packages/stack/src/namespace/Registry.ts b/packages/stack/src/namespace/Registry.ts index a34f45cd3e..9346ab8933 100644 --- a/packages/stack/src/namespace/Registry.ts +++ b/packages/stack/src/namespace/Registry.ts @@ -1,11 +1,15 @@ -import { Effect, Exit, FileSystem, Path, Schedule, Schema, Scope } from "effect"; +import { Clock, Effect, Exit, FileSystem, Option, Path, Schedule, Schema, Scope } from "effect"; import { CompositionConfig } from "../Orchestrator.ts"; import { ServiceCreation } from "../services/Catalog.ts"; import { namespaceError, retryTransientRead, type NamespaceError } from "./Capabilities.ts"; import { OWNER_FILE, OWNER_LOG_FILE } from "./Lease.ts"; import * as Publication from "./Publication.ts"; +import { removeEmptyDirectory } from "./drivers/FileSystem.ts"; import { acquireLock, isBusy, takeLock } from "./drivers/Sqlite.ts"; +/** How long a destroyed stack's empty directories stay, so a same-id restart finds them. */ +const DESTROYED_RETENTION_MILLIS = 10 * 60 * 1000; + const SafeId = Schema.String.pipe( Schema.refine((value): value is string => /^[a-zA-Z0-9_-]+$/u.test(value), { identifier: "SafeStateId", @@ -80,6 +84,8 @@ export interface Interface { readonly list: Effect.Effect, NamespaceError>; readonly save: (state: SavedStack) => Effect.Effect; readonly remove: (id: string) => Effect.Effect; + /** Removes the empty directories `remove` left for stacks destroyed over ten minutes ago, other than `keep`. */ + readonly pruneDestroyed: (keep: string) => Effect.Effect; /** Not reentrant; wrap metadata updates here, while Ports operations acquire this lock themselves. */ readonly withLock: ( effect: Effect.Effect, @@ -186,6 +192,39 @@ export const make = ( // The stack and data directories stay: deleting and recreating them for a restart under the // same id lets the container engine's file share report fresh bind sources as missing. }); + const isDestroyedShell = Effect.fnUntraced(function* (id: string, now: number) { + const directory = stackRoot(id); + if ((yield* fs.stat(directory)).type !== "Directory") return false; + const entries = yield* fs.readDirectory(directory); + if (entries.length !== 1 || entries[0] !== "data") return false; + const data = path.join(directory, "data"); + if ((yield* fs.stat(data)).type !== "Directory") return false; + if ((yield* fs.readDirectory(data)).length > 0) return false; + // Read after the contents, so a destroy that just emptied the directory counts as recent. + const modified = Option.getOrUndefined((yield* fs.stat(directory)).mtime)?.getTime(); + return modified !== undefined && now - modified >= DESTROYED_RETENTION_MILLIS; + }); + const pruneDestroyed = Effect.fn("Namespace.Registry.pruneDestroyed")(function* (keep: string) { + const now = yield* Clock.currentTimeMillis; + let removed = 0; + for (const id of yield* stackIds) { + if (id === keep) continue; + const pruned = yield* Effect.gen(function* () { + if (!(yield* isDestroyedShell(id, now))) return false; + yield* removeEmptyDirectory(path.join(stackRoot(id), "data")); + yield* removeEmptyDirectory(stackRoot(id)); + return true; + }).pipe( + Effect.catch((error) => + Effect.logWarning(`Could not prune destroyed stack directory ${id}`, error).pipe( + Effect.as(false), + ), + ), + ); + if (pruned) removed += 1; + } + yield* Effect.annotateCurrentSpan({ removed }); + }); /** * One attempt: forks `attemptScope` under `guardScope` before opening anything, so there is * always a scope to close if the open or the `BEGIN IMMEDIATE` below fails or defects. Runs @@ -236,5 +275,5 @@ export const make = ( ), ); - return { read, list: list(), save, remove, withLock }; + return { read, list: list(), save, remove, pruneDestroyed, withLock }; }); From f98d0916cede9d256041f15b1f529e289ff42bc3 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 07:47:14 +0200 Subject: [PATCH 071/142] fix(stack): stop services in dependency order without draining listeners first --- .oxlintrc.json | 2 +- ...7-simplified-managed-stack-architecture.md | 2 +- packages/stack/ARCHITECTURE.md | 39 ++- .../stack/src/HttpProxy.integration.test.ts | 209 +------------- packages/stack/src/HttpProxy.ts | 50 +--- .../stack/src/Network.integration.test.ts | 186 ------------ packages/stack/src/Network.ts | 95 +----- .../src/Orchestrator.integration.test.ts | 110 +++++++ packages/stack/src/Orchestrator.ts | 26 +- packages/stack/src/Owner.ts | 22 +- packages/stack/src/Proxy.integration.test.ts | 45 --- packages/stack/src/Proxy.ts | 16 +- ...ost.container-shutdown.integration.test.ts | 270 ++++-------------- .../stack/src/StackHost.integration.test.ts | 167 ----------- packages/stack/src/effect.integration.test.ts | 115 +++++++- ...ixture.ts => gated-docker-stop-fixture.ts} | 30 +- 16 files changed, 344 insertions(+), 1040 deletions(-) rename packages/stack/tests/{gated-drain-fixture.ts => gated-docker-stop-fixture.ts} (68%) diff --git a/.oxlintrc.json b/.oxlintrc.json index 36f4b46dbb..95028bdf47 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -374,7 +374,7 @@ "packages/stack/tests/host-process-fixture.ts", "packages/stack/tests/failing-owner-fixture.ts", "packages/stack/tests/short-registration-poll-fixture.ts", - "packages/stack/tests/gated-drain-fixture.ts", + "packages/stack/tests/gated-docker-stop-fixture.ts", "packages/stack/tests/watch-entry.ts" ], "rules": { diff --git a/docs/adr/0017-simplified-managed-stack-architecture.md b/docs/adr/0017-simplified-managed-stack-architecture.md index f9ee11209f..234243f38f 100644 --- a/docs/adr/0017-simplified-managed-stack-architecture.md +++ b/docs/adr/0017-simplified-managed-stack-architecture.md @@ -68,7 +68,7 @@ Disabling a capability releases its automatic port assignment; re-enabling it ma select a new port. Stack handles are lightweight identity-scoped clients. Creating or opening one -does not launch a Supervisor. Successful stop drains ingress, removes every +does not launch a Supervisor. Successful stop stops services in dependency order, removes every ephemeral runtime resource, persists stopped state, delivers its response, then closes control and releases ownership. The caller waits for both response and lease release. A stopped stack therefore consumes no live process, container, diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index fa0fcff8a9..1abdd4acce 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -495,7 +495,7 @@ is deleted, moved or unmounted). Every owner polls its own `//sta interval (30 seconds by default) and treats only a confirmed ENOENT as gone; any other read error, such as a permission failure or a transient I/O error on an unmounted root, keeps the owner running and logs a warning instead. On a confirmed-gone registration the owner treats its -ownership as ended: it drains (connection drain, above), stops every workload and removes what it +ownership as ended: it stops every workload (ordered stop, above) and removes what it created through the same registration-independent `cleanupResources` path destroy uses, retrying transient cleanup failures and giving up only once an engine is confirmed unreachable (the leftover container ids are logged, not retried forever), releases its port reservations, and exits. It never @@ -512,32 +512,31 @@ Where the platform delivers SIGTERM or SIGINT to the host, treat it as the same During Draining: -1. Close admission to new mutations and proxy wake requests. +1. Close admission to new mutations and to proxy traffic that would wake a service at rest. + Traffic to starting or running services stays admitted. 2. Reject queued work that has not begun. -3. Drain every stack listener, public and dependency alike (connection drain, below). -4. Let executing instance operations settle. -5. Cancel and settle attached commands and stop owned services through their existing operations. -6. For destruction, remove proven-owned data and metadata after shutdown. -7. Send the outcome, close the control endpoint and release ownership. - -**Connection drain.** Stop and destroy share one drain step, run before any service stops: accept -closes on every stack listener, public and internal dependency traffic alike (they are the same -listeners), so no new connection is admitted, and a request or upgrade arriving on an established HTTP -connection is refused (503 with `Connection: close`, or a destroyed socket). Drain then waits, for at most 10 seconds, only for -observable work: HTTP requests in flight and upgraded sockets such as WebSockets. TCP connections -never count, so an idle pooled connection (PostgREST's database pool, for example) cannot delay -shutdown. Services then stop, and listener scopes close afterward through the ordinary -stop/destroy teardown, which cuts whatever remains. Accepted, documented tradeoff: during the -drain window, an admitted request that needs a new dependency connection can fail; the deadline -bounds that window. +3. Let executing instance operations settle. +4. Cancel and settle attached commands and stop owned services in reverse dependency order, + with every listener still open (ordered stop, below). +5. For destruction, remove proven-owned data and metadata after shutdown. +6. Send the outcome, close the control endpoint and release ownership. + +**Ordered stop.** Stop, destroy and abandonment close no listener up front. Public and internal +dependency traffic share the same listeners, so a dependent's graceful stop (Vector flushing its +last batch to Analytics, for example) still reaches its prerequisite through the stack proxy while +it is running. Services stop in reverse dependency order, and each listener closes once its own +service has stopped, through the ordinary scope teardown. Traffic that would wake a service at rest +is refused for the whole shutdown. Accepted, documented tradeoff: the proxy offers no completion +window, so an in-flight request survives only as far as its service's own graceful stop carries it, +and a slow client can lose a response. **Draining is one-way.** The host holds the single serving/draining fact; the owner reads it for -admission and the network keeps only its listener accept gate. Once a stop or destroy begins, the +admission. Once a stop or destroy begins, the owner never serves again. A failed stop or destroy reports its error, the stack stays registered with uncertain state, and the owner exits. The next stop or destroy finds no owner and reclaims the stack under its lease. -**Successful whole-stack shutdown.** `stack.stop()` reports success only after admitted work settles, every owned live service and command workload has stopped (including native processes, descendants and containers labeled with this stack and data root), stack listeners close, the shutdown request is acknowledged, and the detached host's exit is confirmed. If workload cleanup cannot be confirmed, stop fails and the host exits without resuming service; the stack is not reported stopped and remains registered for the next stop or destroy to reclaim. If cleanup succeeds but host exit cannot be confirmed, stop also fails, though the host may already have exited. A delivered SIGTERM or SIGINT follows this cleanup path. Stop preserves stack definitions, service data, caches and port reservations. `destroy` follows the same live-workload cleanup, then removes only proven-owned persistent data. +**Successful whole-stack shutdown.** `stack.stop()` reports success only after admitted work settles, every owned live service and command workload has stopped (including native processes, descendants and containers labeled with this stack and data root), stack listeners close with their services, the shutdown request is acknowledged, and the detached host's exit is confirmed. If workload cleanup cannot be confirmed, stop fails and the host exits without resuming service; the stack is not reported stopped and remains registered for the next stop or destroy to reclaim. If cleanup succeeds but host exit cannot be confirmed, stop also fails, though the host may already have exited. A delivered SIGTERM or SIGINT follows this cleanup path. Stop preserves stack definitions, service data, caches and port reservations. `destroy` follows the same live-workload cleanup, then removes only proven-owned persistent data. The client captures the live owner PID from the validated identity endpoint or readiness handshake, completes the shutdown request, then performs bounded process-existence checks. An absent PID confirms exit; a permission-denied probe remains inconclusive until the deadline. Failure to confirm exit is a `shutdown-exit` error carrying the PID in its message, even when workload cleanup has already succeeded. This does not require persisted PID records or forceful termination. Caller cancellation ends its wait without cancelling admitted owner cleanup. diff --git a/packages/stack/src/HttpProxy.integration.test.ts b/packages/stack/src/HttpProxy.integration.test.ts index e6bb9de550..11adb430dc 100644 --- a/packages/stack/src/HttpProxy.integration.test.ts +++ b/packages/stack/src/HttpProxy.integration.test.ts @@ -1,6 +1,6 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Data, Deferred, Effect, Fiber, Layer, Stream, SubscriptionRef } from "effect"; +import { Data, Deferred, Effect, Fiber, Layer } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { createServer, type Server, type ServerResponse } from "node:http"; // oxlint-disable-line effecttsgo/node-builtin-import -- raw server fixture. import { createServer as createTcpServer, Socket, type Server as NetServer } from "node:net"; // oxlint-disable-line effecttsgo/node-builtin-import -- raw disconnect fixture. @@ -1092,210 +1092,3 @@ it.live("releases a waiting WebSocket target quietly when its client resets", () }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, captureErrors(logs)))); }); - -const connectRaw = (port: number, host: string) => - Effect.acquireRelease( - Effect.callback((resume) => { - const socket = new Socket(); - const onConnect = () => resume(Effect.succeed(socket)); - const onError = (cause: Error) => - resume(Effect.fail(new HttpProxyTestError({ message: cause.message, cause }))); - socket.once("connect", onConnect); - socket.once("error", onError); - socket.connect(port, host); - return Effect.sync(() => { - socket.off("connect", onConnect); - socket.off("error", onError); - }); - }).pipe(Effect.timeout("5 seconds")), - (socket) => Effect.sync(() => socket.destroy()), - ); - -/** Reads one full HTTP/1.1 response off `socket`: accumulates chunks until the declared body length. */ -const readFullResponse = (socket: Socket) => - Effect.callback<{ readonly status: number; readonly body: string }, HttpProxyTestError>( - (resume) => { - let buffered = Buffer.alloc(0); - let headerEnd: number | undefined; - let status = 0; - let contentLength = 0; - const onData = (chunk: Buffer) => { - buffered = Buffer.concat([buffered, chunk]); - if (headerEnd === undefined) { - const index = buffered.indexOf("\r\n\r\n"); - if (index === -1) return; - headerEnd = index + 4; - const head = buffered.subarray(0, headerEnd).toString("latin1"); - status = Number(/^HTTP\/1\.1 (\d+)/u.exec(head)?.[1] ?? 0); - contentLength = Number(/content-length:\s*(\d+)/iu.exec(head)?.[1] ?? 0); - } - if (buffered.length - headerEnd < contentLength) return; - socket.off("data", onData); - resume( - Effect.succeed({ - status, - body: buffered.subarray(headerEnd, headerEnd + contentLength).toString(), - }), - ); - }; - socket.on("data", onData); - return Effect.sync(() => socket.off("data", onData)); - }, - ).pipe(Effect.timeout("5 seconds")); - -/** - * Resolves once a fresh connection attempt to `port` is refused rather than served: the listener - * accepts the TCP handshake but destroys the socket immediately, so no response ever arrives. - */ -const connectionRefused = (port: number, host: string) => - Effect.acquireRelease( - Effect.sync(() => new Socket()), - (socket) => Effect.sync(() => socket.destroy()), - ).pipe( - Effect.flatMap((probe) => - Effect.callback((resume) => { - const onSettled = (responded: boolean) => resume(Effect.succeed(!responded)); - probe.once("error", () => onSettled(false)); - probe.once("close", () => onSettled(false)); - probe.once("data", () => onSettled(true)); - probe.connect(port, host, () => - probe.write("GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"), - ); - return Effect.void; - }).pipe(Effect.timeout("5 seconds")), - ), - ); - -it.live( - "lets a response already in flight when stopAccepting runs finish and deliver its full body", - () => - Effect.scoped( - Effect.gen(function* () { - const backend = createServer((_incoming, outgoing) => { - outgoing.writeHead(200, { "content-type": "text/plain" }); - outgoing.write("first-chunk"); - // Held open deliberately: the test itself finishes the response, once stopAccepting - // has already run, so the response's generation crosses the freeze. - }); - const backendAddress = yield* listen(backend); - const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); - yield* proxy.setRoutes([ - { id: "api", prefix: "/", target: Effect.succeed(backendAddress) }, - ]); - - const heldResponseFiber = yield* Effect.callback( - (resume) => { - const onRequest = (_incoming: unknown, outgoing: ServerResponse) => - resume(Effect.succeed(outgoing)); - backend.once("request", onRequest); - return Effect.sync(() => backend.off("request", onRequest)); - }, - ).pipe(Effect.forkScoped); - const responseFiber = yield* request(proxy.port, "/", new Uint8Array()).pipe( - Effect.provide(NodeHttpClient.layerNodeHttp), - Effect.forkScoped, - ); - const heldResponse = yield* Fiber.join(heldResponseFiber); - - yield* proxy.stopAccepting; - expect(yield* connectionRefused(proxy.port, proxy.host)).toBe(true); - - heldResponse?.end("-second-chunk"); - const response = yield* Fiber.join(responseFiber); - expect(response.status).toBe(200); - expect(new TextDecoder().decode(response.body)).toBe("first-chunk-second-chunk"); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), -); - -it.live( - "closes an idle established connection once stopAccepting runs, while still refusing new connections", - () => - Effect.scoped( - Effect.gen(function* () { - const backend = createServer((_request, response) => response.end("first-response")); - const backendAddress = yield* listen(backend); - const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); - yield* proxy.setRoutes([ - { id: "api", prefix: "/", target: Effect.succeed(backendAddress) }, - ]); - - const client = yield* connectRaw(proxy.port, proxy.host); - client.write("GET / HTTP/1.1\r\nHost: localhost\r\nConnection: keep-alive\r\n\r\n"); - const first = yield* readFullResponse(client); - expect(first.status).toBe(200); - expect(first.body).toBe("first-response"); - // The request already finished, so the connection sits idle with no active work. - expect(yield* SubscriptionRef.get(proxy.outstandingConnections)).toBe(0); - - const closed = yield* Effect.callback((resume) => { - const onClose = () => resume(Effect.void); - client.once("close", onClose); - if (client.destroyed) onClose(); - return Effect.sync(() => client.off("close", onClose)); - }).pipe(Effect.timeout("5 seconds"), Effect.forkScoped); - - yield* proxy.stopAccepting; - - expect(yield* connectionRefused(proxy.port, proxy.host)).toBe(true); - yield* Fiber.join(closed); - }), - ).pipe(Effect.provide(NodeServices.layer)), -); - -it.live( - "answers a request submitted on an established connection after stopAccepting with 503 while the in-flight response completes", - () => - Effect.scoped( - Effect.gen(function* () { - let received = 0; - const backend = createServer((_incoming, outgoing) => { - received += 1; - outgoing.writeHead(200, { "content-length": "9" }); - // Held open: the test finishes this response after the late request has been sent. - }); - const backendAddress = yield* listen(backend); - const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); - yield* proxy.setRoutes([ - { id: "api", prefix: "/", target: Effect.succeed(backendAddress) }, - ]); - - const heldResponseFiber = yield* Effect.callback( - (resume) => { - const onRequest = (_incoming: unknown, outgoing: ServerResponse) => - resume(Effect.succeed(outgoing)); - backend.once("request", onRequest); - return Effect.sync(() => backend.off("request", onRequest)); - }, - ).pipe(Effect.forkScoped); - const client = yield* connectRaw(proxy.port, proxy.host); - const closed = yield* Effect.callback((resume) => { - const chunks: Array = []; - client.on("data", (chunk: Buffer) => chunks.push(chunk)); - client.once("close", () => resume(Effect.succeed(Buffer.concat(chunks).toString()))); - return Effect.void; - }).pipe(Effect.timeout("5 seconds"), Effect.forkScoped); - client.write("GET /slow HTTP/1.1\r\nHost: localhost\r\nConnection: keep-alive\r\n\r\n"); - const heldResponse = yield* Fiber.join(heldResponseFiber); - expect(yield* SubscriptionRef.get(proxy.outstandingConnections)).toBe(1); - - yield* proxy.stopAccepting; - client.write("GET /late HTTP/1.1\r\nHost: localhost\r\nConnection: keep-alive\r\n\r\n"); - heldResponse.end("slow-body"); - - const transcript = yield* Fiber.join(closed); - const inFlight = transcript.indexOf("HTTP/1.1 200"); - const late = transcript.indexOf("HTTP/1.1 503"); - expect(inFlight).toBeGreaterThanOrEqual(0); - expect(late).toBeGreaterThan(inFlight); - expect(transcript.slice(inFlight, late)).toContain("slow-body"); - expect(transcript.slice(late).toLowerCase()).toContain("connection: close"); - expect(received).toBe(1); - yield* SubscriptionRef.changes(proxy.outstandingConnections).pipe( - Stream.filter((count) => count === 0), - Stream.runHead, - Effect.timeout("5 seconds"), - ); - }), - ).pipe(Effect.provide(NodeServices.layer)), -); diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index d45f22c157..35f08e4aca 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -1,4 +1,4 @@ -import { Data, Effect, FiberSet, Ref, Scope, SubscriptionRef } from "effect"; +import { Data, Effect, FiberSet, Ref, Scope } from "effect"; import { PortError } from "./Ports.ts"; import type { BackendAddress, ProxyError } from "./Proxy.ts"; import { @@ -45,13 +45,6 @@ export interface HttpProxy { readonly host: string; readonly port: number; readonly setRoutes: (routes: ReadonlyArray) => Effect.Effect; - /** Stops accepting new connections and closes idle ones; in-flight requests keep flowing. */ - readonly stopAccepting: Effect.Effect; - /** - * The count of connections with active work, observable until it reaches 0: a request in - * flight or an upgraded socket, never an idle keep-alive connection. - */ - readonly outstandingConnections: SubscriptionRef.SubscriptionRef; } const hopByHop = new Set([ @@ -428,25 +421,8 @@ export const makeHttpProxy = (options: { (value) => Effect.sync(() => value.destroy()), ); const runRequest = yield* FiberSet.makeRuntime(); - const services = yield* Effect.context(); const sockets = new Set(); - const outstandingConnections = yield* SubscriptionRef.make(0); - // `server.close()` also tears down active connections immediately, not just idle ones, so - // draining tracks acceptance separately and closes idle sockets explicitly instead. - let accepting = true; - const closeNowIdle = () => { - if (!accepting) server.closeIdleConnections(); - }; const server = createServer((request, response) => { - if (!accepting) { - response.statusCode = 503; - response.setHeader("Connection", "close"); - response.end("Service Unavailable"); - return; - } - Effect.runSyncWith(services)( - SubscriptionRef.update(outstandingConnections, (count) => count + 1), - ); runRequest( Effect.scoped( Effect.gen(function* () { @@ -481,20 +457,10 @@ export const makeHttpProxy = (options: { ); } }), - ).pipe( - Effect.ensuring( - SubscriptionRef.update(outstandingConnections, (count) => count - 1).pipe( - Effect.andThen(Effect.sync(closeNowIdle)), - ), - ), ), ); }); server.on("connection", (socket) => { - if (!accepting) { - socket.destroy(); - return; - } sockets.add(socket); socket.once("close", () => { sockets.delete(socket); @@ -502,13 +468,6 @@ export const makeHttpProxy = (options: { }); server.on("upgrade", (request, socket, head) => { socket.on("error", () => socket.destroy()); - if (!accepting) { - socket.destroy(); - return; - } - Effect.runSyncWith(services)( - SubscriptionRef.update(outstandingConnections, (count) => count + 1), - ); runRequest( Effect.scoped( Effect.gen(function* () { @@ -526,8 +485,6 @@ export const makeHttpProxy = (options: { Effect.catch(() => Effect.sync(() => socket.destroy())), ); }), - ).pipe( - Effect.ensuring(SubscriptionRef.update(outstandingConnections, (count) => count - 1)), ), ); }); @@ -557,10 +514,5 @@ export const makeHttpProxy = (options: { routes, [...next].sort((a, b) => b.prefix.length - a.prefix.length), ), - stopAccepting: Effect.sync(() => { - accepting = false; - server.closeIdleConnections(); - }), - outstandingConnections, }; }); diff --git a/packages/stack/src/Network.integration.test.ts b/packages/stack/src/Network.integration.test.ts index 57f96dfb42..8e0719b623 100644 --- a/packages/stack/src/Network.integration.test.ts +++ b/packages/stack/src/Network.integration.test.ts @@ -594,189 +594,3 @@ it.live("addresses docker runtime endpoints through the stack host alias", () => }), ).pipe(Effect.provide(NodeServices.layer)), ); - -it.live("drains past an idle HTTP keep-alive connection without ever reaching the deadline", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-drain-idle-" }); - const state = yield* makeTestState(root); - yield* state.save(stack("stack")); - const target = yield* backend; - const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); - const service = yield* network.register({ - id: "studio", - endpoints: { http: endpoint(target, Effect.succeed(true)) }, - }); - yield* service.bind; - const address = yield* service.address("http", "host"); - - // A served keep-alive request proves the listener accepted and tracks this socket, so its - // closure is observed directly instead of by reprobing the port. - const socket = yield* Effect.callback((resume) => { - const connection = Net.createConnection({ host: address.host, port: address.port }); - connection.once("connect", () => { - connection.write( - `GET /hello HTTP/1.1\r\nHost: ${address.host}:${address.port}\r\nConnection: keep-alive\r\n\r\n`, - ); - }); - connection.once("data", () => resume(Effect.succeed(connection))); - connection.on("error", (cause) => - resume(Effect.fail(new FixtureError({ message: cause.message }))), - ); - return Effect.sync(() => connection.destroy()); - }); - const closed = yield* Effect.callback((resume) => { - socket.once("close", () => resume(Effect.void)); - return Effect.void; - }).pipe(Effect.forkChild); - - // A deadline that never fires proves drain resolved through quiescence alone: if the idle - // connection still counted as outstanding, this would hang instead of completing. - yield* network.drain.pipe( - Effect.provideService(Network.ShutdownDrainDeadline, Effect.never), - Effect.timeout("5 seconds"), - ); - yield* Fiber.join(closed).pipe(Effect.timeout("2 seconds")); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), -); - -it.live("drains past an idle TCP connection without ever reaching the deadline", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-drain-tcp-" }); - const state = yield* makeTestState(root); - yield* state.save(stack("stack")); - const target = yield* backend; - const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); - const service = yield* network.register({ - id: "db", - endpoints: { sql: { ...endpoint(target, Effect.succeed(true)), protocol: "tcp" } }, - }); - yield* service.bind; - const address = yield* service.address("sql", "host"); - // A keep-alive round trip through the TCP endpoint proves the listener accepted and tracks - // this connection; the socket then sits idle through drain. - const socket = yield* Effect.callback((resume) => { - const connection = Net.createConnection({ host: address.host, port: address.port }); - connection.once("connect", () => - connection.write( - `GET /hello HTTP/1.1\r\nHost: ${address.host}:${address.port}\r\nConnection: keep-alive\r\n\r\n`, - ), - ); - connection.once("data", () => resume(Effect.succeed(connection))); - connection.on("error", (cause) => - resume(Effect.fail(new FixtureError({ message: cause.message }))), - ); - return Effect.sync(() => connection.destroy()); - }); - - // A deadline that never fires proves TCP never counts toward drain: an idle pooled - // connection stays open and drain still completes. - yield* network.drain.pipe( - Effect.provideService(Network.ShutdownDrainDeadline, Effect.never), - Effect.timeout("5 seconds"), - ); - expect(socket.destroyed).toBe(false); - }), - ).pipe(Effect.provide(NodeServices.layer)), -); - -it.live( - "lets an in-flight HTTP request finish during drain, then closes its now-idle connection", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-drain-inflight-" }); - const state = yield* makeTestState(root); - yield* state.save(stack("stack")); - const services = yield* Effect.context(); - const requested = yield* Deferred.make(); - const release = yield* Deferred.make(); - const held = Effect.acquireRelease( - Effect.callback< - { host: string; port: number; server: ReturnType }, - FixtureError - >((resume) => { - const server = createServer((_request, response) => { - Effect.runForkWith(services)( - Deferred.succeed(requested, undefined).pipe( - Effect.andThen(Deferred.await(release)), - Effect.andThen(Effect.sync(() => response.end("backend:held"))), - ), - ); - }); - const onError = (cause: Error) => - resume(Effect.fail(new FixtureError({ message: cause.message }))); - server.once("error", onError); - server.listen(0, "127.0.0.1", () => { - const address = server.address(); - if (address === null || typeof address === "string") - resume(Effect.fail(new FixtureError({ message: "no address" }))); - else resume(Effect.succeed({ host: "127.0.0.1", port: address.port, server })); - }); - return Effect.sync(() => { - server.off("error", onError); - server.close(); - }); - }), - ({ server }) => - Effect.callback((resume) => { - server.close(() => resume(Effect.void)); - return Effect.void; - }), - ); - const target = yield* held; - const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); - const service = yield* network.register({ - id: "studio", - endpoints: { http: endpoint(target, Effect.succeed(true)) }, - }); - yield* service.bind; - const address = yield* service.address("http", "host"); - - let body = ""; - const received = yield* Deferred.make(); - const socket = yield* Effect.callback((resume) => { - const connection = Net.createConnection({ host: address.host, port: address.port }); - connection.once("connect", () => { - connection.write( - `GET /hello HTTP/1.1\r\nHost: ${address.host}:${address.port}\r\nConnection: keep-alive\r\n\r\n`, - ); - }); - connection.on("data", (bytes) => { - body += bytes.toString(); - if (body.includes("backend:held")) - Effect.runForkWith(services)(Deferred.succeed(received, undefined)); - }); - connection.once("connect", () => resume(Effect.succeed(connection))); - connection.on("error", (cause) => - resume(Effect.fail(new FixtureError({ message: cause.message }))), - ); - return Effect.sync(() => connection.destroy()); - }); - const closed = yield* Effect.callback((resume) => { - socket.once("close", () => resume(Effect.void)); - return Effect.void; - }).pipe(Effect.forkChild); - yield* Deferred.await(requested).pipe(Effect.timeout("2 seconds")); - - // A deadline that never fires proves drain is genuinely waiting on the request in flight, - // not racing a timer, while the held response keeps it from resolving prematurely. - const drained = yield* network.drain.pipe( - Effect.provideService(Network.ShutdownDrainDeadline, Effect.never), - Effect.forkChild, - ); - yield* Deferred.succeed(release, undefined); - // Waiting for the response body itself, not just the drain fiber, proves completion is - // observed through the client's own socket rather than assumed once the fiber joins. - yield* Deferred.await(received).pipe(Effect.timeout("5 seconds")); - yield* Fiber.join(drained).pipe(Effect.timeout("5 seconds")); - expect(body).toContain("backend:held"); - yield* Fiber.join(closed).pipe(Effect.timeout("2 seconds")); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), -); diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index 380224480a..8dfe45ad8f 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -1,15 +1,4 @@ -import { - Context, - Data, - Effect, - Exit, - Layer, - Ref, - Scope, - Semaphore, - Stream, - SubscriptionRef, -} from "effect"; +import { Context, Data, Effect, Exit, Layer, Ref, Scope, Semaphore } from "effect"; import { DOCKER_HOST_ALIAS } from "./runtime/Container.ts"; import * as PortReservations from "./namespace/PortReservations.ts"; import * as StackNamespace from "./StackNamespace.ts"; @@ -19,21 +8,6 @@ import { makeHttpProxy, type HttpProxy, type HttpRoute } from "./HttpProxy.ts"; export type NetworkRuntime = "native" | "docker"; -/** - * The shutdown-drain deadline: in-flight HTTP work keeps flowing until this event completes. - * An event rather than a duration so a test can gate exactly when it fires. - */ -export const ShutdownDrainDeadline = Context.Reference>( - "@supabase/stack/ShutdownDrainDeadline", - { defaultValue: () => Effect.sleep("10 seconds") }, -); - -/** What shutdown drain needs from a listener: stop accepting, and for HTTP, the in-flight work to wait on. */ -interface ListenerHandle { - readonly stopAccepting: Effect.Effect; - readonly outstandingConnections?: SubscriptionRef.SubscriptionRef; -} - type RouteContribution = Pick< HttpRoute, "prefix" | "upstreamPrefix" | "upstreamHost" | "keyRewrite" @@ -86,14 +60,6 @@ export interface Interface { readonly id: string; readonly endpoints: Readonly>; }) => Effect.Effect; - /** - * Shutdown drain, one-way: closes accept on every stack listener, public and dependency alike, - * then waits until in-flight HTTP requests and upgraded sockets settle or `ShutdownDrainDeadline` - * elapses, whichever comes first. TCP connections are never waited on. Established connections - * stay open and listener scopes are untouched; the caller stops the services and closes the - * scopes afterward, which destroys whatever remains. - */ - readonly drain: Effect.Effect; } export class Service extends Context.Service()("@supabase/stack/Network") {} @@ -118,29 +84,6 @@ const makeNetwork = (options: { const probe = probeVacant(options.platform ?? process.platform); const owner = yield* Scope.Scope; const gate = yield* Semaphore.make(1); - // Set once under `gate` by `drain`, before it snapshots `listeners`, and never cleared; checked - // under the same gate by `bind`, so a bind that hasn't yet acquired the gate when drain starts - // either completes before drain's snapshot (and so is captured by it) or observes this and - // refuses outright so neither slips a new, untracked listener past both the snapshot and the - // accept cut. - const draining = yield* Ref.make(false); - // Every bound listener (the shared API proxy under "api", every dedicated endpoint under - // "id:name"), tracked only for shutdown drain; registration and deregistration are tied to the - // same scope that owns the listener, so this never drifts from what is actually bound. - const listeners = yield* Ref.make>(new Map()); - const trackListener = (key: string, handle: ListenerHandle) => - Ref.update(listeners, (current) => new Map(current).set(key, handle)).pipe( - Effect.andThen( - Effect.addFinalizer(() => - Ref.update(listeners, (current) => { - if (current.get(key) !== handle) return current; - const next = new Map(current); - next.delete(key); - return next; - }), - ), - ), - ); const shared = yield* Ref.make< | { readonly claim: number; @@ -211,7 +154,6 @@ const makeNetwork = (options: { Effect.gen(function* () { if (yield* Ref.get(closed)) return yield* errorFor("bind", "Network namespace is closed"); - if (yield* Ref.get(draining)) return yield* errorFor("bind", "Network is draining"); for (const [name, endpoint] of Object.entries(endpoints)) { if ((yield* Ref.get(bound)).has(name)) continue; yield* Effect.uninterruptibleMask((restore) => @@ -236,7 +178,6 @@ const makeNetwork = (options: { } yield* probe(key, host, port); const proxy = yield* makeHttpProxy({ host, port }); - yield* trackListener(key, proxy); return { proxy }; } if (endpoint.protocol === "http") { @@ -245,12 +186,10 @@ const makeNetwork = (options: { yield* proxy.setRoutes([ { id, prefix: "/", target: endpoint.backend }, ]); - yield* trackListener(key, proxy); return { proxy }; } yield* probe(key, host, port); const listener = yield* bindTcp(host, port); - yield* trackListener(key, { stopAccepting: listener.stopAccepting }); yield* Effect.forkIn( serveTcp(listener, endpoint.backend, `${id}:${name}`).pipe( Effect.provideService(Scope.Scope, endpointScope), @@ -403,41 +342,9 @@ const makeNetwork = (options: { .releaseStack(options.stackId) .pipe(Effect.mapError((cause) => errorFor("release", cause))), ); - // Stopping accepting can only shrink each HTTP listener's outstanding count from here on, so - // waiting for every handle to independently reach 0 is equivalent to waiting for the total. - const quiesced = (count: SubscriptionRef.SubscriptionRef) => - SubscriptionRef.changes(count).pipe( - Stream.takeUntil((outstanding) => outstanding === 0), - Stream.runDrain, - ); - const drain = Effect.fn("Network.drain")(function* () { - const deadline = yield* ShutdownDrainDeadline; - // Setting `draining` and snapshotting `listeners` atomically under the same gate `bind` - // holds for its whole call means no bind can complete after this snapshot without either - // being captured by it or observing `draining` and refusing before creating anything. - const handles = yield* gate.withPermits(1)( - Ref.set(draining, true).pipe( - Effect.andThen(Ref.get(listeners)), - Effect.map((current) => [...current.values()]), - ), - ); - yield* Effect.forEach(handles, (handle) => handle.stopAccepting, { discard: true }); - const inFlight = handles.flatMap((handle) => - handle.outstandingConnections === undefined ? [] : [handle.outstandingConnections], - ); - const outcome = yield* Effect.forEach(inFlight, quiesced, { - concurrency: "unbounded", - discard: true, - }).pipe( - Effect.as("quiesced" as const), - Effect.race(deadline.pipe(Effect.as("deadline" as const))), - ); - yield* Effect.annotateCurrentSpan({ listeners: handles.length, outcome }); - }); return { register, releaseStack: releaseStack(), - drain: drain(), } satisfies Interface; }); diff --git a/packages/stack/src/Orchestrator.integration.test.ts b/packages/stack/src/Orchestrator.integration.test.ts index 067f41b0b8..b5b1b57722 100644 --- a/packages/stack/src/Orchestrator.integration.test.ts +++ b/packages/stack/src/Orchestrator.integration.test.ts @@ -1187,6 +1187,116 @@ describe("wake and idle sleep", () => { }), ).pipe(Effect.provide(TestClock.layer())), ); + + describe("while the owner refuses new work", () => { + const makeRefusingOrchestrator = Effect.gen(function* () { + const draining = yield* Ref.make(false); + const orchestrator = yield* Orchestrator.make({ + admit: () => + Ref.get(draining).pipe( + Effect.flatMap((refusing) => + refusing ? Effect.fail(failure("Owner is draining")) : Effect.void, + ), + ), + }); + return { orchestrator, draining }; + }); + + it.live( + "refuses traffic that would wake a stopped lazy service, then wakes once admitted", + () => + Effect.scoped( + Effect.gen(function* () { + const { orchestrator, draining } = yield* makeRefusingOrchestrator; + const api = yield* makeInstance(orchestrator, "api"); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + + yield* Ref.set(draining, true); + const refused = yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); + expect(refused.message).toBe("Owner is draining"); + expect(yield* Ref.get(api.starts)).toHaveLength(0); + expect((yield* api.status).lifecycle).toBe("stopped"); + + yield* Ref.set(draining, false); + yield* Effect.scoped(orchestrator.acquire("api")); + expect(yield* Ref.get(api.starts)).toHaveLength(1); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live( + "refuses traffic that arrives while an idle stop is in progress, without relaunching", + () => + Effect.scoped( + Effect.gen(function* () { + const { orchestrator, draining } = yield* makeRefusingOrchestrator; + const stopping = yield* Deferred.make(); + const finishStop = yield* Deferred.make(); + const api = yield* makeInstance(orchestrator, "api", { + stop: Deferred.succeed(stopping, undefined).pipe( + Effect.andThen(Deferred.await(finishStop)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy", idleMillis: 1000 }], + dependencies: [], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("api")); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(stopping); + + yield* Ref.set(draining, true); + const refused = yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); + expect(refused.message).toBe("Owner is draining"); + + yield* Deferred.succeed(finishStop, undefined); + yield* orchestrator.stopNamespace; + expect(yield* Ref.get(api.starts)).toHaveLength(1); + expect((yield* api.status).lifecycle).toBe("stopped"); + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live("still admits traffic to a starting or running service", () => + Effect.scoped( + Effect.gen(function* () { + const { orchestrator, draining } = yield* makeRefusingOrchestrator; + const launching = yield* Deferred.make(); + const launch = yield* Deferred.make(); + const api = yield* makeInstance(orchestrator, "api", { + launch: Deferred.succeed(launching, undefined).pipe( + Effect.andThen(Deferred.await(launch)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const waking = yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.forkChild); + yield* Deferred.await(launching); + + yield* Ref.set(draining, true); + const joining = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(launch, undefined); + yield* Fiber.join(waking); + yield* Fiber.join(joining); + yield* Effect.scoped(orchestrator.acquire("api")); + expect(yield* Ref.get(api.starts)).toHaveLength(1); + expect((yield* api.status).lifecycle).toBe("running"); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + }); }); describe("idle races through a real listener", () => { diff --git a/packages/stack/src/Orchestrator.ts b/packages/stack/src/Orchestrator.ts index 33756fc40b..d6ec25ff61 100644 --- a/packages/stack/src/Orchestrator.ts +++ b/packages/stack/src/Orchestrator.ts @@ -150,8 +150,11 @@ export interface Status extends ServiceObservation { readonly destroyPending: boolean; } -/** An explicit operation the owner may refuse, for example while it drains. */ -export type AdmittedOperation = "start" | "restart" | "storage"; +/** + * An operation the owner may refuse, for example while it drains. `wake` is traffic that would + * relaunch a service at rest. + */ +export type AdmittedOperation = "start" | "restart" | "storage" | "wake"; /** The lifecycle authority for one stack's registered instances and their composition. */ export interface Interface { @@ -396,7 +399,7 @@ const cleanupError = (id: string, cause: unknown) => /** Builds an orchestrator whose lifecycle actor, timers and executions live in the current scope. */ export const make = Effect.fn("Orchestrator.make")(function* ( options: { - /** Refuses an explicit start, restart or storage operation, checked under the lifecycle gate. */ + /** Refuses a start, restart, storage operation or traffic wake, checked under the lifecycle gate. */ readonly admit?: (operation: AdmittedOperation) => Effect.Effect; } = {}, ): Effect.fn.Return, never, Scope.Scope> { @@ -639,7 +642,10 @@ export const make = Effect.fn("Orchestrator.make")(function* >; readonly waits: ReadonlyArray; - /** Traffic waits carry a lease and the wake budget; explicit waits carry neither. */ + /** + * Traffic waits carry a lease and the wake budget; explicit waits carry neither. Traffic that + * finds its service neither starting nor running is admitted as a `wake`. + */ readonly traffic: boolean; /** Checks the owner's admission guard under the same gate step as the requests. */ readonly operation?: AdmittedOperation; @@ -706,8 +712,18 @@ export const make = Effect.fn("Orchestrator.make")(function* { + const phase = state.services.get(id)?.phase._tag; + return phase !== "Starting" && phase !== "Running"; + }); + if (wakes) yield* admit("wake"); + }); const guarded = withGate( - (admission.operation === undefined ? Effect.void : admit(admission.operation)).pipe( + admitUnderGate.pipe( Effect.andThen(Ref.set(reached, true)), Effect.andThen( applyLocked([ diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index 94278de676..d9120f680d 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -142,7 +142,7 @@ export interface Interface { */ readonly destroy: Effect.Effect; /** - * Ends ownership after a confirmed-gone registration: drains, stops every workload, and + * Ends ownership after a confirmed-gone registration: stops every workload, and * removes what it created through the same registration-independent cleanup path as destroy, * without touching the registration or releasing port reservations. A leftover resource past * the engine-unreachable backstop is logged, not thrown. @@ -785,18 +785,16 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { handlers, getStackCredentials, namespace: { - // Shutdown drain: accept closes on every listener before any service stops, and the - // listener scopes only close afterward, through stop/destroy's ordinary teardown below. - stop: network.drain.pipe( - Effect.andThen(orchestrator.stopNamespace), + // Services stop in reverse dependency order with their listeners open, so a dependent's + // graceful stop can still reach its prerequisites through the proxy. + stop: orchestrator.stopNamespace.pipe( Effect.andThen(sweep), definitionGate.withPermits(1), Effect.withSpan("Owner.stopNamespace"), ), // Instance teardown retains port rows until the registration is gone; the release after // that is best-effort because `isGone` reclaims a gone stack's rows lazily. - destroy: network.drain.pipe( - Effect.andThen(orchestrator.destroyNamespace), + destroy: orchestrator.destroyNamespace.pipe( Effect.andThen(sweep), Effect.andThen(removeContainerEnvRoot), Effect.andThen(options.state.remove(stackId)), @@ -813,13 +811,9 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { // Settles admitted definition work under the same gate as stop/destroy first. Never // reads or writes the registration (already confirmed gone), so every step below is // best-effort and logs rather than fails; the caller must still be able to exit. - abandon: network.drain.pipe( - Effect.andThen( - orchestrator.stopNamespace.pipe( - Effect.catch((cause) => - Effect.logWarning("Abandoned stack could not confirm every workload stopped", cause), - ), - ), + abandon: orchestrator.stopNamespace.pipe( + Effect.catch((cause) => + Effect.logWarning("Abandoned stack could not confirm every workload stopped", cause), ), Effect.andThen( orchestrator.instances.pipe( diff --git a/packages/stack/src/Proxy.integration.test.ts b/packages/stack/src/Proxy.integration.test.ts index db13c06afa..3f3b396b80 100644 --- a/packages/stack/src/Proxy.integration.test.ts +++ b/packages/stack/src/Proxy.integration.test.ts @@ -248,51 +248,6 @@ it.live( ), ); -it.live( - "refuses a new connection after stopAccepting while an established connection keeps relaying data through the scoped target", - () => - Effect.scoped( - Effect.gen(function* () { - const backend = yield* echoBackend(); - const listener = yield* bindTcp("127.0.0.1", 0); - if (!Predicate.isTagged(listener.address, "TcpAddress")) - return yield* Effect.die("Expected TCP listener"); - const target = Effect.succeed({ host: backend.host, port: backend.port }); - yield* serveTcp(listener, target, "echo").pipe(Effect.forkScoped); - const port = listener.address.port; - - const client = yield* connectAndWrite(port, "ping"); - expect(yield* readOnceTcp(client)).toBe("ping"); - - yield* listener.stopAccepting; - - const refused = yield* Effect.callback((resume) => { - const probe = new Socket(); - let responded = false; - let settled = false; - // A refused connection surfaces as a reset, a generic error, or a plain close; all three - // are the allowed refusal outcomes here, so long as no response ever arrived. - const settle = () => { - if (settled) return; - settled = true; - resume(Effect.succeed(!responded)); - }; - probe.once("data", () => { - responded = true; - }); - probe.once("error", settle); - probe.once("close", settle); - probe.connect(port, "127.0.0.1", () => probe.write("probe")); - return Effect.sync(() => probe.destroy()); - }).pipe(Effect.timeout("5 seconds")); - expect(refused).toBe(true); - - client.write("pong"); - expect(yield* readOnceTcp(client)).toBe("pong"); - }), - ), -); - it.live("tears down a connection that arrives before run installs its handler", () => Effect.gen(function* () { const listenerScope = yield* Scope.make(); diff --git a/packages/stack/src/Proxy.ts b/packages/stack/src/Proxy.ts index 9bd17a95a6..5aaa583abb 100644 --- a/packages/stack/src/Proxy.ts +++ b/packages/stack/src/Proxy.ts @@ -22,15 +22,11 @@ export interface TcpListener { readonly run: ( handler: (socket: Net.Socket) => Effect.Effect<_, E, R>, ) => Effect.Effect; - /** Stops accepting new connections on the listener; established connections keep flowing. */ - readonly stopAccepting: Effect.Effect; } /** - * Binds a dedicated public listener and retains the socket until its scope closes. Built directly - * on `node:net`, not `NodeSocketServer`, because `NodeSocketServer`'s `run` only exposes a combined - * accept-and-destroy teardown: this separates closing accept (`stopAccepting`) from destroying - * established connections, which happens when the scope closes. + * Binds a dedicated public listener and retains the socket until its scope closes, which also + * destroys every established connection. */ export const bindTcp = ( host: string, @@ -41,7 +37,6 @@ export const bindTcp = ( // Every accepted socket, queued or dispatched, from acceptance until close, so teardown // destroys all of them. const sockets = new Set(); - let accepting = true; // A connection that arrives before `run` installs its handler is queued, the same as // `NodeSocketServer`'s own listener, so none are dropped in that gap. const pending = new Set(); @@ -52,10 +47,6 @@ export const bindTcp = ( // Attached synchronously, before any dispatch, so a reset while a handler is still cold // (queued, or waiting on scoped target acquisition) never surfaces as an unhandled error. conn.on("error", () => conn.destroy()); - if (!accepting) { - conn.destroy(); - return; - } sockets.add(conn); conn.once("close", () => { sockets.delete(conn); @@ -133,9 +124,6 @@ export const bindTcp = ( return { address, run, - stopAccepting: Effect.sync(() => { - accepting = false; - }), }; }); diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index a4c4ff7882..e0caa24273 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -4,11 +4,9 @@ import { expect, it } from "@effect/vitest"; import { Context, Crypto, - Data, Effect, Exit, FileSystem, - Fiber, Layer, Path, Redacted, @@ -17,7 +15,7 @@ import { } from "effect"; import * as Reactivity from "effect/unstable/reactivity/Reactivity"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import * as Net from "node:net"; // oxlint-disable-line effecttsgo/node-builtin-import -- forces an outstanding connection during a gated drain. +import * as Net from "node:net"; // oxlint-disable-line effecttsgo/node-builtin-import -- holds a raw TCP connection open across the owner stop. import { fileURLToPath } from "node:url"; import * as StackNamespace from "./StackNamespace.ts"; import { @@ -36,100 +34,10 @@ import { watchEntry } from "../tests/watch-entry.ts"; const shortRegistrationPollFixture = fileURLToPath( new URL("../tests/short-registration-poll-fixture.ts", import.meta.url), ); -const gatedDrainFixture = fileURLToPath( - new URL("../tests/gated-drain-fixture.ts", import.meta.url), +const gatedDockerStopFixture = fileURLToPath( + new URL("../tests/gated-docker-stop-fixture.ts", import.meta.url), ); -class HostTestError extends Data.TaggedError("HostTestError")<{ readonly message: string }> {} - -const hostTestError = (cause: unknown) => - new HostTestError({ message: cause instanceof Error ? cause.message : String(cause) }); - -const isHttpRefused = (port: number) => - Effect.callback((resume) => { - let socket: Net.Socket; - try { - socket = Net.connect(port, "127.0.0.1"); - } catch { - resume(Effect.succeed(true)); - return Effect.void; - } - let response = ""; - const onConnect = () => - socket.write("GET / HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n"); - const onData = (bytes: Buffer) => (response += bytes.toString()); - const settle = () => resume(Effect.succeed(response.length === 0)); - socket.once("connect", onConnect); - socket.on("data", onData); - socket.once("close", settle); - socket.once("error", settle); - return Effect.sync(() => { - socket.off("connect", onConnect); - socket.off("data", onData); - socket.off("close", settle); - socket.off("error", settle); - socket.destroy(); - }); - }); - -/** - * Opens a real TCP connection to the mail service's send endpoint and writes only the first half - * of the request body, so the request stays genuinely in flight until `complete` sends the rest. - */ -const openPartialMailSend = (port: number, subject: string) => - Effect.gen(function* () { - // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- raw wire payload split mid-body to hold the request open, not a domain model - const payload = JSON.stringify({ - From: { Email: "sender@example.com" }, - To: [{ Email: "recipient@example.com" }], - Subject: subject, - Text: "drain test", - }); - const splitAt = Math.ceil(payload.length / 2); - const socket = yield* Effect.acquireRelease( - Effect.callback((resume) => { - let socket: Net.Socket; - try { - socket = Net.connect(port, "127.0.0.1"); - } catch (cause) { - resume(Effect.fail(hostTestError(cause))); - return Effect.void; - } - const onConnect = () => - socket.write( - `POST /api/v1/send HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Type: application/json\r\nContent-Length: ${payload.length}\r\nConnection: close\r\n\r\n${payload.slice(0, splitAt)}`, - () => resume(Effect.succeed(socket)), - ); - const onError = (cause: Error) => resume(Effect.fail(hostTestError(cause))); - socket.once("connect", onConnect); - socket.once("error", onError); - return Effect.sync(() => { - socket.off("connect", onConnect); - socket.off("error", onError); - }); - }), - (socket) => Effect.sync(() => socket.destroy()), - ); - const response = yield* Effect.callback((resume) => { - let buffer = ""; - const onData = (bytes: Buffer) => (buffer += bytes.toString()); - const onEnd = () => resume(Effect.succeed(buffer)); - const onError = (cause: Error) => resume(Effect.fail(hostTestError(cause))); - socket.on("data", onData); - socket.once("end", onEnd); - socket.once("error", onError); - return Effect.sync(() => { - socket.off("data", onData); - socket.off("end", onEnd); - socket.off("error", onError); - }); - }).pipe(Effect.forkChild({ startImmediately: true })); - return { - complete: () => socket.write(payload.slice(splitAt)), - response: Fiber.join(response), - }; - }); - const helperImage = "public.ecr.aws/docker/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; @@ -516,19 +424,56 @@ it.live.skipIf(process.platform === "win32")( { timeout: 180_000 }, ); +/** + * Installs a `docker` shim the owner finds first on its PATH: the first `docker stop` announces + * itself by creating `waiting` in `gateDir` and blocks until the test writes the `release` FIFO, + * then every command runs the real docker. + */ +const holdDockerStops = (gateDir: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + yield* fs.makeDirectory(`${gateDir}/bin`, { recursive: true }); + yield* spawner.spawn(ChildProcess.make("mkfifo", [`${gateDir}/release`])).pipe( + Effect.flatMap((child) => child.exitCode), + Effect.scoped, + ); + yield* fs.writeFileString( + `${gateDir}/bin/docker`, + [ + "#!/bin/sh", + `gate='${gateDir}'`, + 'if [ ! -e "$gate/passed" ]; then', + ' for arg in "$@"; do', + ' if [ "$arg" = stop ]; then', + ' : > "$gate/waiting"', + ' read -r _ < "$gate/release"', + ' : > "$gate/passed"', + " break", + " fi", + " done", + "fi", + 'PATH="${PATH#"$gate/bin:"}"', + 'exec docker "$@"', + "", + ].join("\n"), + { mode: 0o755 }, + ); + }); + type StopTrigger = "signal" | "http"; /** * Starts a data-preserving stop through `trigger` without waiting for it to finish: a * signal-driven stop runs in the owner process on its own, while an HTTP-driven stop is forked - * so the gated drain it blocks on never blocks the caller. + * so the held container stop it blocks on never blocks the caller. */ const triggerStop = (trigger: StopTrigger, access: HostAccess) => trigger === "signal" ? Effect.sync(() => process.kill(access.endpoint.pid, "SIGTERM")) : Effect.forkScoped(shutdownOwner(access, false).pipe(Effect.ignore)); -const abandonsWhileStopDrains = (trigger: StopTrigger) => +const abandonsWhileStopRuns = (trigger: StopTrigger) => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -542,7 +487,7 @@ const abandonsWhileStopDrains = (trigger: StopTrigger) => const stateRoot = path.dirname(path.dirname(dataRoot)); const cacheRoot = `${base}/cache`; const gateDir = `${base}/gate`; - yield* fs.makeDirectory(gateDir, { recursive: true }); + yield* holdDockerStops(gateDir); const state = yield* stateFor(stateRoot); yield* state.save({ id: stackId, @@ -561,7 +506,7 @@ const abandonsWhileStopDrains = (trigger: StopTrigger) => stateRoot, cacheRoot, stackId, - entrypoint: gatedDrainFixture, + entrypoint: gatedDockerStopFixture, entrypointArgs: [gateDir], }); const client = yield* ownerClient(access); @@ -580,36 +525,20 @@ const abandonsWhileStopDrains = (trigger: StopTrigger) => const status = yield* client.status({ id: database.id }); if (!status.endpoints.some((endpoint) => endpoint.name === "sql")) return yield* Effect.die("Missing database sql endpoint"); - const mail = yield* client.createService({ - service: "mail", - config: {}, - endpoints: { http: { port: "auto" } }, - }); - yield* client.startService({ id: mail.id }); - yield* client.readyService({ id: mail.id }); - const mailPort = (yield* client.status({ id: mail.id })).endpoints.find( - (endpoint) => endpoint.name === "http", - )?.port; - if (mailPort === undefined) return yield* Effect.die("Missing mail http endpoint"); const containerEnvRoot = `${dataRoot}/.container-env`; expect( yield* fs.exists(containerEnvRoot), "the stack's shared container-env scratch directory exists", ).toBe(true); - // Keeps an HTTP request in flight for the whole gated window, so drain can only resolve - // through the deadline this test controls, never through the work settling on its own. - const held = yield* openPartialMailSend(mailPort, "abandon-gated-inflight"); - - // Subscribes to the drain-deadline wait's own entry marker before triggering the stop, so - // the registration deletion below never races the gate itself. + // Subscribes to the held `docker stop`'s own marker before triggering the stop, so the + // registration deletion below never races the gate itself. const waiting = yield* watchEntry(gateDir, "waiting", true); yield* triggerStop(trigger, access); yield* waiting.pipe(Effect.timeout("30 seconds")); yield* fs.remove(`${stateRoot}/${stackId}/state.json`); yield* fs.writeFileString(`${gateDir}/release`, ""); - held.complete(); yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( Effect.retry({ while: hasReason("owner-exit-pending") }), @@ -629,14 +558,14 @@ const abandonsWhileStopDrains = (trigger: StopTrigger) => ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))); it.live.skipIf(process.platform === "win32")( - "abandons a stack whose registration disappears while a signal-driven stop is still draining", - () => abandonsWhileStopDrains("signal"), + "abandons a stack whose registration disappears while a signal-driven stop is still running", + () => abandonsWhileStopRuns("signal"), { timeout: 180_000 }, ); it.live.skipIf(process.platform === "win32")( - "abandons a stack whose registration disappears while an HTTP-driven stop is still draining", - () => abandonsWhileStopDrains("http"), + "abandons a stack whose registration disappears while an HTTP-driven stop is still running", + () => abandonsWhileStopRuns("http"), { timeout: 180_000 }, ); @@ -836,106 +765,27 @@ it.live.skipIf(process.platform === "win32")( ); it.live.skipIf(process.platform === "win32")( - "drains an in-flight mail request through a real docker owner shutdown, and refuses a new connection once draining begins", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-drain-docker-" }); - const stackId = `drain-docker-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; - const dataRoot = yield* makeDockerDatabaseRoot("stack-drain-docker-data-", stackId).pipe( - Effect.flatMap(fs.realPath), - ); - const stateRoot = path.dirname(path.dirname(dataRoot)); - const cacheRoot = `${base}/cache`; - const gateDir = `${base}/gate`; - yield* fs.makeDirectory(gateDir, { recursive: true }); - const state = yield* stateFor(stateRoot); - yield* state.save({ - id: stackId, - runtime: "docker", - identity: { - projectRoot: `${base}/project`, - branchContext: "drain-docker-test", - stackName: stackId, - }, - instances: [], - lifetime: "detached", - composition: { members: [], dependencies: [] }, - }); - yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); - const access = yield* launchHost(state, { - stateRoot, - cacheRoot, - stackId, - entrypoint: gatedDrainFixture, - entrypointArgs: [gateDir], - }); - const client = yield* ownerClient(access); - const mail = yield* client.createService({ - service: "mail", - config: {}, - endpoints: { http: { port: "auto" } }, - }); - yield* client.startService({ id: mail.id }); - yield* client.readyService({ id: mail.id }); - const status = yield* client.status({ id: mail.id }); - const port = status.endpoints.find((endpoint) => endpoint.name === "http")?.port; - if (port === undefined) return yield* Effect.die("Missing mail http endpoint"); - - const inFlight = yield* openPartialMailSend(port, "drain-docker-inflight"); - - const waiting = yield* watchEntry(gateDir, "waiting", true); - process.kill(access.endpoint.pid, "SIGTERM"); - yield* waiting.pipe(Effect.timeout("30 seconds")); - - expect(yield* isHttpRefused(port), "a new connection is refused once draining begins").toBe( - true, - ); - - inFlight.complete(); - const response = yield* inFlight.response.pipe(Effect.timeout("10 seconds")); - expect( - response.startsWith("HTTP/1.1 200"), - "the in-flight request still completes during drain", - ).toBe(true); - - yield* fs.writeFileString(`${gateDir}/release`, ""); - yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( - Effect.retry({ while: hasReason("owner-exit-pending") }), - Effect.timeout("30 seconds"), - ); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), - { timeout: 180_000 }, -); - -it.live.skipIf(process.platform === "win32")( - "closes a pinned postgres connection and exits a real owner without waiting for the drain deadline", + "closes a pinned postgres connection and exits a real owner promptly", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; - const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-drain-postgres-" }); - const stackId = `drain-postgres-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; - const dataRoot = yield* makeDockerDatabaseRoot("stack-drain-postgres-data-", stackId).pipe( + const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-pinned-postgres-" }); + const stackId = `pinned-postgres-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const dataRoot = yield* makeDockerDatabaseRoot("stack-pinned-postgres-data-", stackId).pipe( Effect.flatMap(fs.realPath), ); const stateRoot = path.dirname(path.dirname(dataRoot)); const cacheRoot = `${base}/cache`; - const gateDir = `${base}/gate`; - yield* fs.makeDirectory(gateDir, { recursive: true }); const state = yield* stateFor(stateRoot); yield* state.save({ id: stackId, runtime: "docker", identity: { projectRoot: `${base}/project`, - branchContext: "drain-postgres-test", + branchContext: "pinned-postgres-test", stackName: stackId, }, instances: [], @@ -947,17 +797,15 @@ it.live.skipIf(process.platform === "win32")( stateRoot, cacheRoot, stackId, - entrypoint: gatedDrainFixture, - entrypointArgs: [gateDir], }); const client = yield* ownerClient(access); - const password = Redacted.make("drain-postgres-password"); + const password = Redacted.make("pinned-postgres-password"); const database = yield* client.createService({ service: "database", config: { version: "17", databasePassword: password, - jwtSecret: Redacted.make("drain-postgres-jwt-secret-at-least-thirty-two-characters"), + jwtSecret: Redacted.make("pinned-postgres-jwt-secret-at-least-thirty-two-characters"), jwtExpiry: 3600, }, endpoints: { sql: { port: "auto" } }, @@ -983,8 +831,8 @@ it.live.skipIf(process.platform === "win32")( const sql = Context.get(services, PgClient.PgClient); yield* sql.unsafe("SELECT 1"); - // The gate is never released: only a drain that ignores the pinned TCP connection lets - // the owner finish, and the connection closes once the services stop. + // The owner stops services without waiting on clients, so the pinned connection closes + // with the database container. yield* Effect.callback((resume) => { const probe = Net.createConnection({ host: "127.0.0.1", port }); probe.once("connect", () => { diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts index 5bbb41d780..35c0a05164 100644 --- a/packages/stack/src/StackHost.integration.test.ts +++ b/packages/stack/src/StackHost.integration.test.ts @@ -3,7 +3,6 @@ import { expect, it } from "@effect/vitest"; import { Cause, Context, - Crypto, Data, Deferred, Effect, @@ -21,9 +20,7 @@ import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"; import * as HttpClient from "effect/unstable/http/HttpClient"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- integration observes exact listener closure. import * as Net from "node:net"; -import { fileURLToPath } from "node:url"; import { - hasReason, launchHost, ownerAuthorization, ownerClient, @@ -39,14 +36,9 @@ import { bindControl, makeRuntime } from "./StackHost.ts"; import { shutdownOwner } from "../tests/owner.ts"; import { postgres } from "./Commands.ts"; import * as CommandRunner from "./host/CommandRunner.ts"; -import { watchEntry } from "../tests/watch-entry.ts"; class HostTestError extends Data.TaggedError("HostTestError")<{ readonly message: string }> {} -const gatedDrainFixture = fileURLToPath( - new URL("../tests/gated-drain-fixture.ts", import.meta.url), -); - const stateFor = (root: string) => Effect.gen(function* () { const context = yield* Layer.build(StackNamespace.layer({ root })); @@ -144,91 +136,6 @@ const awaitClosed = (socket: Net.Socket) => return Effect.sync(() => socket.off("close", onClose)); }); -const isHttpRefused = (port: number) => - Effect.callback((resume) => { - let socket: Net.Socket; - try { - socket = Net.connect(port, "127.0.0.1"); - } catch { - resume(Effect.succeed(true)); - return Effect.void; - } - let response = ""; - const onConnect = () => - socket.write("GET / HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n"); - const onData = (bytes: Buffer) => (response += bytes.toString()); - const settle = () => resume(Effect.succeed(response.length === 0)); - socket.once("connect", onConnect); - socket.on("data", onData); - socket.once("close", settle); - socket.once("error", settle); - return Effect.sync(() => { - socket.off("connect", onConnect); - socket.off("data", onData); - socket.off("close", settle); - socket.off("error", settle); - socket.destroy(); - }); - }); - -/** - * Opens a real TCP connection to the mail service's send endpoint and writes only the first half - * of the request body, so the request stays genuinely in flight until `complete` sends the rest. - */ -const openPartialMailSend = (port: number, subject: string) => - Effect.gen(function* () { - // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- raw wire payload split mid-body to hold the request open, not a domain model - const payload = JSON.stringify({ - From: { Email: "sender@example.com" }, - To: [{ Email: "recipient@example.com" }], - Subject: subject, - Text: "drain test", - }); - const splitAt = Math.ceil(payload.length / 2); - const socket = yield* Effect.acquireRelease( - Effect.callback((resume) => { - let socket: Net.Socket; - try { - socket = Net.connect(port, "127.0.0.1"); - } catch (cause) { - resume(Effect.fail(hostTestError(cause))); - return Effect.void; - } - const onConnect = () => - socket.write( - `POST /api/v1/send HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Type: application/json\r\nContent-Length: ${payload.length}\r\nConnection: close\r\n\r\n${payload.slice(0, splitAt)}`, - () => resume(Effect.succeed(socket)), - ); - const onError = (cause: Error) => resume(Effect.fail(hostTestError(cause))); - socket.once("connect", onConnect); - socket.once("error", onError); - return Effect.sync(() => { - socket.off("connect", onConnect); - socket.off("error", onError); - }); - }), - (socket) => Effect.sync(() => socket.destroy()), - ); - const response = yield* Effect.callback((resume) => { - let buffer = ""; - const onData = (bytes: Buffer) => (buffer += bytes.toString()); - const onEnd = () => resume(Effect.succeed(buffer)); - const onError = (cause: Error) => resume(Effect.fail(hostTestError(cause))); - socket.on("data", onData); - socket.once("end", onEnd); - socket.once("error", onError); - return Effect.sync(() => { - socket.off("data", onData); - socket.off("end", onEnd); - socket.off("error", onError); - }); - }).pipe(Effect.forkChild({ startImmediately: true })); - return { - complete: () => socket.write(payload.slice(splitAt)), - response: Fiber.join(response), - }; - }); - const inProcessRuntime = ( owner: Parameters[0] & { readonly draining: Deferred.Deferred }, state: StackNamespace.Interface, @@ -1174,77 +1081,3 @@ it.live("destroys a stack only after an abandoned composition settles", () => }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); - -it.live( - "drains an in-flight mail request through a real owner shutdown, and refuses a new connection once draining begins", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const crypto = yield* Crypto.Crypto; - const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-drain-native-" }); - const stackId = `drain-native-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; - const stateRoot = `${base}/state`; - const cacheRoot = `${base}/cache`; - const gateDir = `${base}/gate`; - yield* fs.makeDirectory(gateDir, { recursive: true }); - const state = yield* stateFor(stateRoot); - yield* state.save({ - id: stackId, - runtime: "native", - identity: { - projectRoot: `${base}/project`, - branchContext: "drain-native-test", - stackName: stackId, - }, - instances: [], - lifetime: "detached", - composition: { members: [], dependencies: [] }, - }); - const access = yield* launchHost(state, { - stateRoot, - cacheRoot, - stackId, - entrypoint: gatedDrainFixture, - entrypointArgs: [gateDir], - }); - const client = yield* ownerClient(access); - const mail = yield* client.createService({ - service: "mail", - config: {}, - endpoints: { http: { port: "auto" } }, - }); - yield* client.startService({ id: mail.id }); - yield* client.readyService({ id: mail.id }); - const status = yield* client.status({ id: mail.id }); - const port = status.endpoints.find((endpoint) => endpoint.name === "http")?.port; - if (port === undefined) return yield* Effect.die("Missing mail http endpoint"); - - const inFlight = yield* openPartialMailSend(port, "drain-native-inflight"); - - // Subscribes to the drain-deadline wait's own entry marker before sending the signal, so - // the refusal check below never races the gate itself. - const waiting = yield* watchEntry(gateDir, "waiting", true); - process.kill(access.endpoint.pid, "SIGTERM"); - yield* waiting.pipe(Effect.timeout("30 seconds")); - - expect(yield* isHttpRefused(port), "a new connection is refused once draining begins").toBe( - true, - ); - - inFlight.complete(); - const response = yield* inFlight.response.pipe(Effect.timeout("10 seconds")); - expect( - response.startsWith("HTTP/1.1 200"), - "the in-flight request still completes during drain", - ).toBe(true); - - yield* fs.writeFileString(`${gateDir}/release`, ""); - yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( - Effect.retry({ while: hasReason("owner-exit-pending") }), - Effect.timeout("30 seconds"), - ); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), - { timeout: 60_000 }, -); diff --git a/packages/stack/src/effect.integration.test.ts b/packages/stack/src/effect.integration.test.ts index d2392eda1a..a065de7fef 100644 --- a/packages/stack/src/effect.integration.test.ts +++ b/packages/stack/src/effect.integration.test.ts @@ -31,6 +31,7 @@ import * as StackNamespace from "./StackNamespace.ts"; import { assertOwnerExited, watchLeaseRelease } from "../tests/owner.ts"; import { foreignRelease } from "../tests/release-owner-fixture.ts"; import { destroyTestStack } from "../tests/stack-cleanup.ts"; +import { watchEntry } from "../tests/watch-entry.ts"; import { deriveStackId, resolveStackIdentity } from "./identity/Identity.ts"; const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); @@ -213,7 +214,7 @@ it.live( ); it.live( - "stops a running database within the drain deadline while a client holds an idle TCP connection", + "stops a running database promptly while a client holds an idle TCP connection", () => Effect.scoped( Effect.gen(function* () { @@ -252,7 +253,7 @@ it.live( return Effect.sync(() => connection.destroy()); }); - // The drain deadline is 10 seconds; an idle client must not make stop wait for it. + // An idle client must not hold the stop open. yield* stack.stop.pipe(Effect.timeout("5 seconds")); client.destroy(); @@ -1150,3 +1151,113 @@ it.live("plans a project's own URL for an input whose supplying member is absent ); }).pipe(Effect.scoped, Effect.provide(layer)), ); + +it.live( + "lets a dependent finish its final flush through the stack proxy while the stack stops", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-stop-order-" }); + const options = { + projectRoot: root, + stateRoot: `${root}/state`, + cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + runtime: "native", + } satisfies Parameters[0]; + const secret = "stop-order-secret-with-at-least-thirty-two-characters"; + const seenDirectory = `${root}/seen`; + yield* fs.makeDirectory(seenDirectory, { recursive: true }); + // The analytics batch never fills or times out, so Vector's shutdown flush is the only + // request it sends; the file sink shows when Vector has an event to flush. + const vectorConfigPath = `${root}/vector.yaml`; + yield* fs.writeFileString( + vectorConfigPath, + [ + "sources:", + " demo:", + " type: demo_logs", + " format: json", + " interval: 0.2", + "transforms:", + " marked:", + " type: remap", + " inputs: [demo]", + " source: '.event_message = \"stop-order\"'", + "sinks:", + " seen:", + " type: file", + " inputs: [marked]", + ` path: "${seenDirectory}/events.log"`, + " encoding:", + " codec: json", + " analytics:", + " type: http", + " inputs: [marked]", + ' uri: "${LOGFLARE_URL}/api/logs?source_name=postgres.logs"', + " method: post", + " encoding:", + " codec: json", + " request:", + " headers:", + ' x-api-key: "${LOGFLARE_PRIVATE_ACCESS_TOKEN}"', + " batch:", + " max_events: 100000", + " timeout_secs: 3600", + " healthcheck: false", + "", + ].join("\n"), + ); + const stack = yield* create(options); + yield* Effect.addFinalizer(() => destroyTestStack(stack)); + const services = yield* stack.composition.supabase( + [ + { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("stop-order-password"), + jwtSecret: Redacted.make(secret), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }, + { + service: "analytics", + config: { backend: "postgres", apiKey: secret }, + endpoints: { http: { port: "auto" } }, + }, + { + service: "vector", + config: { apiKey: secret, configPath: vectorConfigPath }, + endpoints: { http: { port: "auto" } }, + }, + ], + { eager: true, keys: { gotrueJwtKeys: "[]", publicSigningKeys: "[]" } }, + ); + const vector = services.find((instance) => instance.service === "vector"); + if (vector === undefined) return yield* Effect.die("vector service missing"); + + const shutDownGracefully = yield* vector.logs.pipe( + Stream.map(({ bytes }) => bytes), + Stream.decodeText, + Stream.splitLines, + Stream.filter((line) => line.includes("All components shut down gracefully")), + Stream.runHead, + // The log stream fails with the owner's exit, which a forced Vector kill delays. + Effect.orElseSucceed(() => Option.none()), + Effect.forkScoped({ startImmediately: true }), + ); + const eventRead = yield* watchEntry(seenDirectory, "events.log", true); + yield* stack.composition.start; + yield* eventRead.pipe(Effect.timeout("60 seconds")); + yield* stack.stop.pipe(Effect.timeout("60 seconds")); + + const shutdown = yield* Fiber.join(shutDownGracefully).pipe(Effect.timeout("10 seconds")); + expect(Option.isSome(shutdown), "vector finished its flush instead of being killed").toBe( + true, + ); + }), + ).pipe(Effect.provide(layer)), + { timeout: 300_000 }, +); diff --git a/packages/stack/tests/gated-drain-fixture.ts b/packages/stack/tests/gated-docker-stop-fixture.ts similarity index 68% rename from packages/stack/tests/gated-drain-fixture.ts rename to packages/stack/tests/gated-docker-stop-fixture.ts index 7229971d5d..ae0be6da0a 100644 --- a/packages/stack/tests/gated-drain-fixture.ts +++ b/packages/stack/tests/gated-docker-stop-fixture.ts @@ -1,28 +1,7 @@ -import { Cause, Effect, Exit, FileSystem, Option, Schema } from "effect"; +import { Cause, Effect, Exit, Option, Schema } from "effect"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- readiness is an inherited launcher descriptor. import { closeSync, writeSync } from "node:fs"; -import * as Network from "../src/Network.ts"; import { runStackHost, StackHostError } from "../src/StackHost.ts"; -import { watchEntry } from "./watch-entry.ts"; - -/** - * A real owner entrypoint, identical to `internal/host-process.ts`, except it overrides - * `Network.ShutdownDrainDeadline` to wait for a `release` file in `gateDir` instead of racing a - * real timer, so a test can hold a shutdown's drain open deterministically from outside the - * subprocess. Production code never reads this from an environment variable or `Config`; only - * this dedicated test fixture supplies a different value. - */ -const awaitRelease = (gateDir: string) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - // Subscribes before signaling "waiting" below, so a `release` written the instant a test - // observes that signal can never land in the gap between the signal and this watch attaching. - const released = yield* watchEntry(gateDir, "release", true); - // Marks entry into the drain-deadline wait itself, so a test can subscribe to this instead - // of a sleep before it is safe to act on the assumption that drain has actually begun. - yield* fs.writeFileString(`${gateDir}/waiting`, ""); - yield* released; - }); const writeLine = (value: unknown) => Effect.gen(function* () { @@ -49,6 +28,10 @@ const writeLine = (value: unknown) => ), ); +/** + * A real owner entrypoint, identical to `internal/host-process.ts`, except it puts + * `/bin` first on PATH so a test can interpose on the `docker` commands the owner runs. + */ const [stateRoot, cacheRoot, stackId, gateDir, ...rest] = process.argv.slice(2); const program = Effect.gen(function* () { @@ -63,6 +46,8 @@ const program = Effect.gen(function* () { operation: "startup", message: "Expected stateRoot, cacheRoot, stackId and gateDir", }); + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the PATH shim must reach the child processes the owner spawns. + process.env.PATH = `${gateDir}/bin:${process.env.PATH ?? ""}`; let reported = false; const report = (value: unknown) => Effect.suspend(() => { @@ -76,7 +61,6 @@ const program = Effect.gen(function* () { stackId, onReady: ({ endpoint, secret }) => report({ type: "ready", endpoint, secret }), }).pipe( - Effect.provideService(Network.ShutdownDrainDeadline, awaitRelease(gateDir)), Effect.catchCause((cause) => { const failure = Option.getOrUndefined(Cause.findErrorOption(cause)); return report({ From f2308833241933c6c3654e2529db3e9b08a94067 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 07:59:02 +0200 Subject: [PATCH 072/142] test(stack): release the docker stop hold from the test scope --- .oxlintrc.json | 1 + ...ost.container-shutdown.integration.test.ts | 20 +++++++++++++++--- packages/stack/tests/release-fifo.ts | 21 +++++++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 packages/stack/tests/release-fifo.ts diff --git a/.oxlintrc.json b/.oxlintrc.json index 95028bdf47..4bb66cf00c 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -375,6 +375,7 @@ "packages/stack/tests/failing-owner-fixture.ts", "packages/stack/tests/short-registration-poll-fixture.ts", "packages/stack/tests/gated-docker-stop-fixture.ts", + "packages/stack/tests/release-fifo.ts", "packages/stack/tests/watch-entry.ts" ], "rules": { diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index e0caa24273..53aceeeb3e 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -28,6 +28,7 @@ import { } from "./HostProcess.ts"; import { makeContainerRuntime, resolveEngineTarget } from "./runtime/Container.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; +import { holdReleaseFifo } from "../tests/release-fifo.ts"; import { shutdownOwner, watchLeaseRelease } from "../tests/owner.ts"; import { watchEntry } from "../tests/watch-entry.ts"; @@ -427,7 +428,7 @@ it.live.skipIf(process.platform === "win32")( /** * Installs a `docker` shim the owner finds first on its PATH: the first `docker stop` announces * itself by creating `waiting` in `gateDir` and blocks until the test writes the `release` FIFO, - * then every command runs the real docker. + * then every command runs the real docker. Returns the idempotent, scope-owned `release`. */ const holdDockerStops = (gateDir: string) => Effect.gen(function* () { @@ -459,6 +460,7 @@ const holdDockerStops = (gateDir: string) => ].join("\n"), { mode: 0o755 }, ); + return yield* holdReleaseFifo(`${gateDir}/release`); }); type StopTrigger = "signal" | "http"; @@ -487,7 +489,7 @@ const abandonsWhileStopRuns = (trigger: StopTrigger) => const stateRoot = path.dirname(path.dirname(dataRoot)); const cacheRoot = `${base}/cache`; const gateDir = `${base}/gate`; - yield* holdDockerStops(gateDir); + const hold = yield* holdDockerStops(gateDir); const state = yield* stateFor(stateRoot); yield* state.save({ id: stackId, @@ -509,6 +511,18 @@ const abandonsWhileStopRuns = (trigger: StopTrigger) => entrypoint: gatedDockerStopFixture, entrypointArgs: [gateDir], }); + // Runs before the container and gate-directory cleanup: frees the held stop, signals the + // owner (a signal during a running stop stays queued), and requires its exit. + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + yield* hold.release; + yield* Effect.try(() => process.kill(access.endpoint.pid, "SIGTERM")).pipe(Effect.ignore); + yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.retry({ while: hasReason("owner-exit-pending") }), + Effect.timeout("30 seconds"), + ); + }).pipe(Effect.orDie), + ); const client = yield* ownerClient(access); const database = yield* client.createService({ service: "database", @@ -538,7 +552,7 @@ const abandonsWhileStopRuns = (trigger: StopTrigger) => yield* waiting.pipe(Effect.timeout("30 seconds")); yield* fs.remove(`${stateRoot}/${stackId}/state.json`); - yield* fs.writeFileString(`${gateDir}/release`, ""); + yield* hold.release; yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( Effect.retry({ while: hasReason("owner-exit-pending") }), diff --git a/packages/stack/tests/release-fifo.ts b/packages/stack/tests/release-fifo.ts new file mode 100644 index 0000000000..346163bb89 --- /dev/null +++ b/packages/stack/tests/release-fifo.ts @@ -0,0 +1,21 @@ +import { Effect } from "effect"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- the FIFO write end must stay open read-write so releasing never blocks. +import { closeSync, constants, openSync, writeSync } from "node:fs"; + +/** + * Holds an existing FIFO open read-write for the scope, so a release never blocks on a missing + * reader and one written before a reader opens is buffered. `release` is idempotent. + */ +export const holdReleaseFifo = (path: string) => + Effect.gen(function* () { + const holder = openSync(path, constants.O_RDWR); + let released = false; + yield* Effect.addFinalizer(() => Effect.sync(() => closeSync(holder))); + return { + release: Effect.sync(() => { + if (released) return; + released = true; + writeSync(holder, "\n"); + }), + }; + }); From c39398330e1b206df608f89bccb1a818a9868bc0 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 09:08:40 +0200 Subject: [PATCH 073/142] test(stack): wait for the owner to exit after its lease is released --- packages/stack/tests/owner.ts | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/packages/stack/tests/owner.ts b/packages/stack/tests/owner.ts index 1678d75ab4..e76297721b 100644 --- a/packages/stack/tests/owner.ts +++ b/packages/stack/tests/owner.ts @@ -1,9 +1,16 @@ import { expect } from "@effect/vitest"; -import { Deferred, Effect, Fiber, Option, Predicate, Stream } from "effect"; +import { Deferred, Effect, Fiber, FileSystem, Option, Stream } from "effect"; import { ChildProcess } from "effect/unstable/process"; import { fileURLToPath } from "node:url"; import { discover, type StackLocations } from "../src/effect.ts"; -import { HostProcessError, shutdownHost, type HostAccess } from "../src/HostProcess.ts"; +import { + hasReason, + HostProcessError, + ownerExitProbe, + shutdownHost, + waitForOwnerExit, + type HostAccess, +} from "../src/HostProcess.ts"; /** Requests shutdown through the owner's stable endpoint, failing with the owner's refusal. */ export const shutdownOwner = Effect.fn("Test.shutdownOwner")(function* ( @@ -72,17 +79,12 @@ export const captureOwnerPid = Effect.fn("Test.captureOwnerPid")(function* ( return owner.pid; }); +/** Waits for the owner to leave the process table: its lease is released before it exits. */ export const assertOwnerExited = Effect.fn("Test.assertOwnerExited")(function* (pid: number) { - const exists = yield* Effect.try({ - try: () => process.kill(pid, 0), - catch: (cause) => - new HostProcessError({ operation: "test-owner", message: String(cause), cause }), - }).pipe( + const fs = yield* FileSystem.FileSystem; + const exited = yield* waitForOwnerExit(pid, ownerExitProbe(fs)).pipe( Effect.as(true), - Effect.catchIf( - ({ cause }) => Predicate.hasProperty(cause, "code") && cause.code === "ESRCH", - () => Effect.succeed(false), - ), + Effect.catchIf(hasReason("owner-exit-pending"), () => Effect.succeed(false)), ); - expect(exists, `Owner process ${pid} survived successful teardown`).toBe(false); + expect(exited, `Owner process ${pid} survived successful teardown`).toBe(true); }); From 9605292fbac03eea921c3654f76a83b3df3fc170 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 09:55:46 +0200 Subject: [PATCH 074/142] fix(stack): let analytics stop without its fifteen second drain delay --- packages/stack/src/effect.integration.test.ts | 19 +++++++++++++++++++ packages/stack/src/services/Analytics.ts | 3 +++ 2 files changed, 22 insertions(+) diff --git a/packages/stack/src/effect.integration.test.ts b/packages/stack/src/effect.integration.test.ts index a065de7fef..d67837ed70 100644 --- a/packages/stack/src/effect.integration.test.ts +++ b/packages/stack/src/effect.integration.test.ts @@ -1248,6 +1248,18 @@ it.live( Effect.orElseSucceed(() => Option.none()), Effect.forkScoped({ startImmediately: true }), ); + const analytics = services.find((instance) => instance.service === "analytics"); + if (analytics === undefined) return yield* Effect.die("analytics service missing"); + // Logflare logs this only once its SIGTERM wait ends, which a forced kill pre-empts. + const analyticsStopped = yield* analytics.logs.pipe( + Stream.map(({ bytes }) => bytes), + Stream.decodeText, + Stream.splitLines, + Stream.filter((line) => line.includes("grace period reached, stopping the app")), + Stream.runHead, + Effect.orElseSucceed(() => Option.none()), + Effect.forkScoped({ startImmediately: true }), + ); const eventRead = yield* watchEntry(seenDirectory, "events.log", true); yield* stack.composition.start; yield* eventRead.pipe(Effect.timeout("60 seconds")); @@ -1257,6 +1269,13 @@ it.live( expect(Option.isSome(shutdown), "vector finished its flush instead of being killed").toBe( true, ); + const analyticsShutdown = yield* Fiber.join(analyticsStopped).pipe( + Effect.timeout("10 seconds"), + ); + expect( + Option.isSome(analyticsShutdown), + "analytics stopped on its own instead of being killed", + ).toBe(true); }), ).pipe(Effect.provide(layer)), { timeout: 300_000 }, diff --git a/packages/stack/src/services/Analytics.ts b/packages/stack/src/services/Analytics.ts index 0c3e334d47..ccdbbe9384 100644 --- a/packages/stack/src/services/Analytics.ts +++ b/packages/stack/src/services/Analytics.ts @@ -51,6 +51,9 @@ export const makeSpec = (): ProcessRecipeSpec => ({ : { LOGFLARE_PRIVATE_ACCESS_TOKEN: creation.config.apiKey }), POSTGRES_BACKEND_URL: databaseUrl, POSTGRES_BACKEND_SCHEMA: "_analytics", + // Logflare waits this period after SIGTERM before stopping (a load-balancer drain delay), + // longer than the stack's stop grace. + ERL_AFLAGS: "-eval application:set_env(logflare,sigterm_shutdown_grace_period_ms,0).", }; }), args: () => Effect.succeed(["start"]), From 756bbe30857d1d12bcbf2658d029580f12d2416e Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 10:40:07 +0200 Subject: [PATCH 075/142] ci(stack): run the stack stress scenarios only on manual dispatch --- .github/workflows/stack-stress.yml | 47 ++++++++++++++++++++++++++++ .github/workflows/test.yml | 50 +++--------------------------- 2 files changed, 51 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/stack-stress.yml diff --git a/.github/workflows/stack-stress.yml b/.github/workflows/stack-stress.yml new file mode 100644 index 0000000000..a8faf97055 --- /dev/null +++ b/.github/workflows/stack-stress.yml @@ -0,0 +1,47 @@ +name: Stack stress + +# Repeats the idle/wake and parallel-stack scenarios back to back on each +# backend to surface timing-dependent races. Manual only: a failing iteration +# fails the job outright and is never retried. +on: + workflow_dispatch: + inputs: + iterations: + description: Runs per backend + type: number + default: 10 + +permissions: + contents: read + +concurrency: + group: stack-stress-${{ github.ref }} + cancel-in-progress: true + +jobs: + stress: + name: Run stack stress scenarios (${{ matrix.runtime }}) + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + runtime: [native, docker] + steps: + - name: Checkout + uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1.0.0-beta + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + go-cache: "false" + + # vitest's --repeats counts the runs after the first. + - name: Run idle and parallel-stack scenarios repeatedly + working-directory: packages/stack + run: >- + pnpm run test:e2e:run + whole-stack.${{ matrix.runtime }}.idle-parallel + --repeats=$((${{ inputs.iterations }} - 1)) + --passWithNoTests=false diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0184dd99e9..3460543918 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -351,47 +351,6 @@ jobs: src/public.e2e.test.ts --passWithNoTests=false - # Repeats the idle/wake and parallel-stack scenarios back to back on both - # backends so timing-dependent races surface before they reach users. A - # failing iteration fails the job outright; it is never retried. - test-stack-stress: - if: | - !startsWith(github.head_ref, 'release-notes/') && - (github.event_name == 'merge_group' || - inputs.force || - github.event.pull_request.draft == false) - name: Run stack stress scenarios - runs-on: blacksmith-8vcpu-ubuntu-2404 - timeout-minutes: 90 - env: - # Runs per backend; vitest's --repeats counts the runs after the first. - STACK_STRESS_ITERATIONS: 10 - steps: - - name: Checkout - uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1.0.0-beta - - - name: Setup - uses: ./.github/actions/setup - with: - dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} - go-cache: "false" - - - name: Run native idle and parallel-stack scenarios repeatedly - working-directory: packages/stack - run: >- - pnpm run test:e2e:run - whole-stack.native.idle-parallel - --repeats=$((STACK_STRESS_ITERATIONS - 1)) - --passWithNoTests=false - - - name: Run Docker idle and parallel-stack scenarios repeatedly - working-directory: packages/stack - run: >- - pnpm run test:e2e:run - whole-stack.docker.idle-parallel - --repeats=$((STACK_STRESS_ITERATIONS - 1)) - --passWithNoTests=false - # Summary job that gates branch protection. It preserves the original check # name and succeeds iff every CLI shard and stack runtime succeeded. test-e2e-summary: @@ -402,16 +361,15 @@ jobs: inputs.force || github.event.pull_request.draft == false) name: Run end-to-end tests - needs: [test-e2e, test-stack-e2e, test-stack-native-scenarios, test-stack-stress] + needs: [test-e2e, test-stack-e2e, test-stack-native-scenarios] runs-on: ubuntu-latest steps: - name: Verify all shards succeeded run: | if [ "${{ needs.test-e2e.result }}" = "failure" ] || [ "${{ needs.test-e2e.result }}" = "cancelled" ] || \ [ "${{ needs.test-stack-e2e.result }}" = "failure" ] || [ "${{ needs.test-stack-e2e.result }}" = "cancelled" ] || \ - [ "${{ needs.test-stack-native-scenarios.result }}" = "failure" ] || [ "${{ needs.test-stack-native-scenarios.result }}" = "cancelled" ] || \ - [ "${{ needs.test-stack-stress.result }}" = "failure" ] || [ "${{ needs.test-stack-stress.result }}" = "cancelled" ]; then - echo "::error ::One or more e2e jobs failed: cli=${{ needs.test-e2e.result }}, stack=${{ needs.test-stack-e2e.result }}, stack-native-scenarios=${{ needs.test-stack-native-scenarios.result }}, stack-stress=${{ needs.test-stack-stress.result }}" + [ "${{ needs.test-stack-native-scenarios.result }}" = "failure" ] || [ "${{ needs.test-stack-native-scenarios.result }}" = "cancelled" ]; then + echo "::error ::One or more e2e jobs failed: cli=${{ needs.test-e2e.result }}, stack=${{ needs.test-stack-e2e.result }}, stack-native-scenarios=${{ needs.test-stack-native-scenarios.result }}" exit 1 fi - echo "All e2e jobs reported: cli=${{ needs.test-e2e.result }}, stack=${{ needs.test-stack-e2e.result }}, stack-native-scenarios=${{ needs.test-stack-native-scenarios.result }}, stack-stress=${{ needs.test-stack-stress.result }}" + echo "All e2e jobs reported: cli=${{ needs.test-e2e.result }}, stack=${{ needs.test-stack-e2e.result }}, stack-native-scenarios=${{ needs.test-stack-native-scenarios.result }}" From 36f7b68774019d227b15eb1123fb05d7633ecf2f Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 11:13:43 +0200 Subject: [PATCH 076/142] fix(stack): reap only staging directories so a preparer's fresh lock survives --- .../stack/src/preparation/ArtifactStore.ts | 10 +++--- .../preparation/artifacts.integration.test.ts | 31 ++++++++++++++++--- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/packages/stack/src/preparation/ArtifactStore.ts b/packages/stack/src/preparation/ArtifactStore.ts index 72b2bafdb8..eb57182ef0 100644 --- a/packages/stack/src/preparation/ArtifactStore.ts +++ b/packages/stack/src/preparation/ArtifactStore.ts @@ -533,15 +533,17 @@ const stagingLockFor = (path: Path.Path, stagingRoot: string, token: string) => /** A staging lock's own file, or the rollback journal SQLite leaves while a transaction is open. */ const STAGING_LOCK_SUFFIX = /\.lock(-journal)?$/u; -/** Every `.staging` token with a directory, a lock file, or both: a reap target either way. */ +/** + * Every `.staging` token with a directory. A bare lock file is skipped: its owner creates the + * lock before taking it and the directory only afterwards, so reaping it could unlink a lock + * about to be taken. + */ const listStagingTokens = ( fs: FileSystem.FileSystem, stagingRoot: string, ): Effect.Effect> => fs.readDirectory(stagingRoot).pipe( - Effect.map((names) => - Array.from(new Set(names.map((name) => name.replace(STAGING_LOCK_SUFFIX, "")))), - ), + Effect.map((names) => names.filter((name) => !STAGING_LOCK_SUFFIX.test(name))), Effect.orElseSucceed(() => []), ); diff --git a/packages/stack/src/preparation/artifacts.integration.test.ts b/packages/stack/src/preparation/artifacts.integration.test.ts index 12651b74bc..8530facd6e 100644 --- a/packages/stack/src/preparation/artifacts.integration.test.ts +++ b/packages/stack/src/preparation/artifacts.integration.test.ts @@ -713,6 +713,29 @@ describe("lock-guarded staging", () => { 20_000, ); + it.live( + "leaves a staging lock file that has no directory yet for the preparer that is about to take it", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-opening-lock-", + }); + const key = "database/postgres-opening-lock"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + yield* store.prepare({ ...request, key }); + const openingLock = `${root}/${key}/.staging/opening-token.lock`; + yield* fs.writeFile(openingLock, new Uint8Array()); + + yield* triggerSweep(root); + + expect(yield* fs.exists(openingLock)).toBe(true); + }), + ), + 20_000, + ); + it.live( "never touches a live preparer's staging directory while another preparer runs", () => @@ -761,9 +784,9 @@ describe("pins", () => { const key = "database/postgres-pin-inprocess"; const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); - const firstScope = yield* Scope.make(); + const firstScope = yield* Scope.fork(yield* Scope.Scope, "sequential"); const firstUse = yield* store.use({ ...request, key }).pipe(Scope.provide(firstScope)); - const secondScope = yield* Scope.make(); + const secondScope = yield* Scope.fork(yield* Scope.Scope, "sequential"); const secondUse = yield* store.use({ ...request, key }).pipe(Scope.provide(secondScope)); expect(firstUse.path).toBe(secondUse.path); @@ -1077,7 +1100,7 @@ describe("pins", () => { const prepared = yield* store.prepare({ ...request, key }); const environment = yield* Environment.confine(fs, path, `${root}/home`); - const processScope = yield* Scope.make(); + const processScope = yield* Scope.fork(yield* Scope.Scope, "sequential"); yield* spawnNativeProcess( { executable: "/bin/sleep", @@ -1121,7 +1144,7 @@ describe("pins", () => { const pidPath = path.join(root, "descendant.pid"); const readyPath = path.join(root, "ready"); const goPath = path.join(root, "go"); - const processScope = yield* Scope.make(); + const processScope = yield* Scope.fork(yield* Scope.Scope, "sequential"); // The descendant ignores SIGTERM and records its own pid, then the leader signals // readiness and waits for the test's own release file before exiting: only the // group-wide SIGKILL in the launcher's exit handler can end the descendant. From f37dfe51bed7abd95d5e132e3cfae83ecbb6e5e7 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 11:15:43 +0200 Subject: [PATCH 077/142] fix(stack): keep the traverse grant on the port registry directory --- packages/stack/src/Ports.integration.test.ts | 61 ++++++++++++++----- .../stack/src/namespace/PortReservations.ts | 8 ++- 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/packages/stack/src/Ports.integration.test.ts b/packages/stack/src/Ports.integration.test.ts index f755e9ab06..a3c1036be4 100644 --- a/packages/stack/src/Ports.integration.test.ts +++ b/packages/stack/src/Ports.integration.test.ts @@ -13,6 +13,7 @@ import { Scope, } from "effect"; import * as Net from "node:net"; +import { userInfo } from "node:os"; import { randomUUID } from "node:crypto"; import { makePorts, @@ -111,30 +112,32 @@ it.live( yield* saveStack(state, root, id); const ports = yield* makePorts(state); const request = { stackId: id, key: "api", host: "127.0.0.1", port: "auto" as const }; - const bind = probedBind("api"); + const probed = probedBind("api"); - const firstScope = yield* Scope.make(); - const first = yield* ports - .acquire(request, bind) - .pipe(Effect.provideService(Scope.Scope, firstScope)); - yield* Scope.close(firstScope, Exit.void); - yield* ports.release(id, "api"); - - // Reacquiring the same stack and key retries the same hash-seeded candidate first, so - // whatever now occupies it deterministically forces the scan onto the next one. + // The scan's first candidate is already reserved when it reaches `bind`; occupying it + // there leaves no gap for a parallel test to take the port before the foreign listener. const foreign = Net.createServer(); - yield* Effect.callback((resume) => { - foreign.once("error", (cause) => resume(Effect.fail(cause))); - foreign.listen(first.port, "127.0.0.1", () => resume(Effect.void)); - }); + let firstCandidate: number | undefined; + const bind = (host: string, port: number) => + Effect.gen(function* () { + if (firstCandidate === undefined) { + firstCandidate = port; + yield* Effect.callback((resume) => { + foreign.once("error", (cause) => resume(Effect.die(cause))); + foreign.listen(port, "127.0.0.1", () => resume(Effect.void)); + }); + } + return yield* probed(host, port); + }); yield* Effect.addFinalizer(() => Effect.callback((resume) => { foreign.close(() => resume(Effect.void)); }), ); - const second = yield* ports.acquire(request, bind); - expect(second.port).not.toBe(first.port); + const acquired = yield* ports.acquire(request, bind); + expect(firstCandidate).toBeDefined(); + expect(acquired.port).not.toBe(firstCandidate); }), ).pipe( Effect.provide( @@ -442,3 +445,29 @@ it.live("reserveNativePort excludes a port a previous attempt lost from the next }), ).pipe(Effect.provide(NodeServices.layer)), ); + +it.live( + "opening the port registry keeps the traverse bit a stepped-down database needs on its directory", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const openRegistry = Layer.build( + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ); + const directory = path.join(userInfo().homedir, ".supabase"); + yield* openRegistry; + // The registry directory is an ancestor of native data, so a root-run Postgres that + // stepped down to another user traverses it through o+x granted by the runtime. + const original = (yield* fs.stat(directory)).mode & 0o7777; + yield* Effect.addFinalizer(() => fs.chmod(directory, original).pipe(Effect.ignore)); + yield* fs.chmod(directory, 0o701); + + yield* openRegistry; + + expect(yield* fs.exists(path.join(directory, "ports.sqlite"))).toBe(true); + expect((yield* fs.stat(directory)).mode & 0o777).toBe(0o701); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/packages/stack/src/namespace/PortReservations.ts b/packages/stack/src/namespace/PortReservations.ts index 05c5f15c55..e8eb458360 100644 --- a/packages/stack/src/namespace/PortReservations.ts +++ b/packages/stack/src/namespace/PortReservations.ts @@ -4,6 +4,7 @@ import { NodeServices } from "@effect/platform-node"; import { Context, Effect, FileSystem, Layer, Path, Predicate, Scope, Stream } from "effect"; import { ChildProcess } from "effect/unstable/process"; import { namespaceError, type NamespaceError } from "./Capabilities.ts"; +import { restrictDirectoryToOwner } from "../runtime/postgres-user.ts"; import { errcode, openDatabase } from "./drivers/Sqlite.ts"; import { lstatPath } from "./drivers/FileSystem.ts"; @@ -99,7 +100,8 @@ const connectionFor = (file: string): Effect.Effect namespaceError("root", cause))); + yield* restrictDirectoryToOwner(fs, directory).pipe( + Effect.mapError((cause) => namespaceError("root", cause)), + ); }); /** `getent passwd ` prints one `name:passwd:uid:gid:gecos:dir:shell` line; field 6 is home. */ From 0a9920ee573f57eaac0ad8267bf61fda489854f1 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 11:15:44 +0200 Subject: [PATCH 078/142] fix(stack): count only recent failures toward the crash-loop breaker --- packages/stack/src/Lifecycle.ts | 14 +++++++--- packages/stack/src/Lifecycle.unit.test.ts | 32 +++++++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/packages/stack/src/Lifecycle.ts b/packages/stack/src/Lifecycle.ts index 231e767638..81ce08deeb 100644 --- a/packages/stack/src/Lifecycle.ts +++ b/packages/stack/src/Lifecycle.ts @@ -3,6 +3,8 @@ import { Data } from "effect"; const waiterCap = 256; const breakerThreshold = 3; const breakerCooldownMillis = 30_000; +/** Failures further apart than this are unrelated; it exceeds the cooldown so a reopened breaker keeps counting. */ +const breakerWindowMillis = 300_000; /** The pause between a failed re-check of a blocking session and the next one. */ const reprobeSpacingMillis = 1_000; @@ -91,12 +93,14 @@ const Phase = Data.taggedEnum(); interface BreakerState { readonly consecutiveFailures: number; + readonly lastFailureAt: number | undefined; readonly openUntil: number | undefined; readonly lastCause: unknown; } const initialBreaker: BreakerState = { consecutiveFailures: 0, + lastFailureAt: undefined, openUntil: undefined, lastCause: undefined, }; @@ -606,14 +610,18 @@ const countFailure = ( now: number, commands: Array, ): BreakerState => { - const consecutiveFailures = service.breaker.consecutiveFailures + 1; + const { lastFailureAt } = service.breaker; + const consecutiveFailures = + lastFailureAt !== undefined && now - lastFailureAt > breakerWindowMillis + ? 1 + : service.breaker.consecutiveFailures + 1; if (consecutiveFailures < breakerThreshold) - return { consecutiveFailures, openUntil: undefined, lastCause: cause }; + return { consecutiveFailures, lastFailureAt: now, openUntil: undefined, lastCause: cause }; const openUntil = now + breakerCooldownMillis; commands.push( LifecycleCommand.ArmCooldownTimer({ id, openUntil, delayMillis: breakerCooldownMillis }), ); - return { consecutiveFailures, openUntil, lastCause: cause }; + return { consecutiveFailures, lastFailureAt: now, openUntil, lastCause: cause }; }; const recordFailure = ( diff --git a/packages/stack/src/Lifecycle.unit.test.ts b/packages/stack/src/Lifecycle.unit.test.ts index ec204e50e8..3b17db2069 100644 --- a/packages/stack/src/Lifecycle.unit.test.ts +++ b/packages/stack/src/Lifecycle.unit.test.ts @@ -458,6 +458,38 @@ describe("Lifecycle reducer", () => { expect(tagsOf(commandsByStep[0]).toSorted()).toEqual(["AdmitConnection", "AdmitConnection"]); }); + it("never opens for failures spaced wider than the window, but a fast start-ready-crash loop still does", () => { + const graph = makeGraph([lazy("flaky")]); + const crashCycle = (initial: LifecycleState, waiterId: number, now: number) => { + let state = initial; + ({ state } = run(state, [{ event: open("flaky", waiterId), now }])); + const generation = startingGeneration(state, "flaky"); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "flaky", generation }), now: now + 1 }, + { + event: LifecycleEvent.Exited({ + id: "flaky", + generation, + cause: "crash", + requested: false, + }), + now: now + 2, + }, + ])); + return state; + }; + + const day = 86_400_000; + let daily = initialState(graph); + for (const waiterId of [1, 2, 3]) daily = crashCycle(daily, waiterId, waiterId * day); + expect(daily.services.get("flaky")?.breaker.openUntil).toBeUndefined(); + expect(daily.services.get("flaky")?.breaker.consecutiveFailures).toBe(1); + + let loop = initialState(graph); + for (const waiterId of [1, 2, 3]) loop = crashCycle(loop, waiterId, waiterId * 1_000); + expect(loop.services.get("flaky")?.breaker.openUntil).toBeDefined(); + }); + it("reopens for the same fixed cooldown on a further failure, and an explicit start resets it", () => { const graph = makeGraph([lazy("flaky")]); let state = initialState(graph); From 36313f8c355bc3a9ced8c9942e28274474a9660f Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 11:15:45 +0200 Subject: [PATCH 079/142] fix(cli): name the stack in destroy retry hints --- apps/cli/src/command-internal/stack-api.ts | 8 ++------ .../src/command-internal/stack-shadow.integration.test.ts | 2 +- apps/cli/src/command-internal/stack-shadow.ts | 2 +- .../experimental/stack/destroy/destroy.handler.ts | 4 ++-- .../destroy.runtime-unavailable.integration.test.ts | 4 ++-- 5 files changed, 8 insertions(+), 12 deletions(-) diff --git a/apps/cli/src/command-internal/stack-api.ts b/apps/cli/src/command-internal/stack-api.ts index 4c52291daf..03d2497a11 100644 --- a/apps/cli/src/command-internal/stack-api.ts +++ b/apps/cli/src/command-internal/stack-api.ts @@ -7,7 +7,6 @@ import { find, open, type CreateOptions, - type DestroyResult, type FindOptions, type FoundStack, type OpenOptions, @@ -68,10 +67,7 @@ export const stackApiLayer = Layer.effect( ).pipe(Layer.provide(FetchHttpClient.layer)); /** Describes why a destroy left engine resources behind, without promising automatic cleanup. */ -export const skippedRuntimeCleanupWarning = ( - subject: string, - _result: Extract, -): string => - `Docker was unavailable, so Docker resources for ${subject} were not removed. Restore Docker and run "supabase stack destroy" again to finish removing it.`; +export const skippedRuntimeCleanupWarning = (subject: string, stackId: string): string => + `Docker was unavailable, so Docker resources for ${subject} were not removed. Restore Docker and run "supabase stack destroy --stack-id ${stackId}" again to finish removing it.`; export type { Stack }; diff --git a/apps/cli/src/command-internal/stack-shadow.integration.test.ts b/apps/cli/src/command-internal/stack-shadow.integration.test.ts index 05a0b08ead..427d23d2a2 100644 --- a/apps/cli/src/command-internal/stack-shadow.integration.test.ts +++ b/apps/cli/src/command-internal/stack-shadow.integration.test.ts @@ -508,7 +508,7 @@ describe("stack shadow databases", () => { ); expect(output.stderrText).toMatch( - /Warning: Docker was unavailable, so Docker resources for shadow stack [0-9a-f]{64} were not removed\. Restore Docker and run "supabase stack destroy" again to finish removing it\.\n/u, + /Warning: Docker was unavailable, so Docker resources for shadow stack [0-9a-f]{64} were not removed\. Restore Docker and run "supabase stack destroy --stack-id [0-9a-f]{64}" again to finish removing it\.\n/u, ); }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), 120_000, diff --git a/apps/cli/src/command-internal/stack-shadow.ts b/apps/cli/src/command-internal/stack-shadow.ts index c6f91e8063..363fa9ccc2 100644 --- a/apps/cli/src/command-internal/stack-shadow.ts +++ b/apps/cli/src/command-internal/stack-shadow.ts @@ -236,7 +236,7 @@ export const stackAcquireShadowDatabase = Effect.fn("StackShadow.acquire")(funct Effect.flatMap((result) => result.runtimeCleanup === "skipped" ? output.raw( - `Warning: ${skippedRuntimeCleanupWarning(`shadow stack ${stack.id}`, result)}\n`, + `Warning: ${skippedRuntimeCleanupWarning(`shadow stack ${stack.id}`, stack.id)}\n`, "stderr", ) : Effect.void, diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts index 936f36acee..be36d020eb 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts @@ -106,7 +106,7 @@ export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( "stack.runtime_cleanup": result.runtimeCleanup, }); if (result.runtimeCleanup === "skipped") - yield* output.warn(skippedRuntimeCleanupWarning(`stack ${target.id}`, result)); + yield* output.warn(skippedRuntimeCleanupWarning(`stack ${target.id}`, target.id)); if (output.format !== "text") yield* output.success("", { destroyed: result.runtimeCleanup === "complete", @@ -117,7 +117,7 @@ export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( yield* output.raw(`Stack ${target.id} destroyed.\n`); else yield* output.raw( - `Stack ${target.id} could not be fully destroyed because Docker is unreachable; restore it and run "supabase stack destroy" again.\n`, + `Stack ${target.id} could not be fully destroyed because Docker is unreachable; restore it and run "supabase stack destroy --stack-id ${target.id}" again.\n`, ); }); return yield* body.pipe(Effect.ensuring(telemetryState.flush)); diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts index 68c8ff8c01..0069203600 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts @@ -71,11 +71,11 @@ it.live( const f = yield* fixture("text"); yield* stackDestroy(f.flags).pipe(Effect.provide(f.layer)); expect(f.output.stdoutText).toBe( - `Stack ${f.stack.id} could not be fully destroyed because Docker is unreachable; restore it and run "supabase stack destroy" again.\n`, + `Stack ${f.stack.id} could not be fully destroyed because Docker is unreachable; restore it and run "supabase stack destroy --stack-id ${f.stack.id}" again.\n`, ); expect(f.output.messages).toContainEqual({ type: "warn", - message: `Docker was unavailable, so Docker resources for stack ${f.stack.id} were not removed. Restore Docker and run "supabase stack destroy" again to finish removing it.`, + message: `Docker was unavailable, so Docker resources for stack ${f.stack.id} were not removed. Restore Docker and run "supabase stack destroy --stack-id ${f.stack.id}" again to finish removing it.`, }); expect(yield* f.api.discover(f.locations)).toHaveLength(1); }).pipe(Effect.scoped, Effect.provide(live)), From 324d3add94dd08f57b33670a3e1114f1fab22b49 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 11:15:45 +0200 Subject: [PATCH 080/142] ci(stack): reject stress iteration counts below one --- .github/workflows/stack-stress.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/stack-stress.yml b/.github/workflows/stack-stress.yml index a8faf97055..3f7cdcb3de 100644 --- a/.github/workflows/stack-stress.yml +++ b/.github/workflows/stack-stress.yml @@ -40,8 +40,14 @@ jobs: # vitest's --repeats counts the runs after the first. - name: Run idle and parallel-stack scenarios repeatedly working-directory: packages/stack - run: >- - pnpm run test:e2e:run - whole-stack.${{ matrix.runtime }}.idle-parallel - --repeats=$((${{ inputs.iterations }} - 1)) - --passWithNoTests=false + env: + ITERATIONS: ${{ inputs.iterations }} + run: | + if ! [ "$ITERATIONS" -ge 1 ] 2>/dev/null; then + echo "iterations must be a whole number of at least 1, got '$ITERATIONS'" >&2 + exit 1 + fi + pnpm run test:e2e:run \ + whole-stack.${{ matrix.runtime }}.idle-parallel \ + --repeats=$((ITERATIONS - 1)) \ + --passWithNoTests=false From d2359533039b057395cc69398f2e7dd256bb73a8 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 11:15:46 +0200 Subject: [PATCH 081/142] docs(stack): align stack docs with ordered stop, breaker and engine support --- .oxlintrc.json | 27 ++++++++++--------- .../stack/destroy/SIDE_EFFECTS.md | 12 ++++----- .../experimental/stack/start/SIDE_EFFECTS.md | 4 +-- .../experimental/stack/stop/SIDE_EFFECTS.md | 10 +++---- ...7-simplified-managed-stack-architecture.md | 9 +++---- ...5-ephemeral-postgres-for-schema-tooling.md | 10 ++++--- packages/stack/ARCHITECTURE.md | 15 +++++------ packages/stack/README.md | 4 +-- packages/stack/src/runtime/Container.ts | 3 --- 9 files changed, 46 insertions(+), 48 deletions(-) diff --git a/.oxlintrc.json b/.oxlintrc.json index 4bb66cf00c..9f4d34677e 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -69,9 +69,10 @@ { // Default for the whole stack package (`src/**` and `tests/**`): on top of the repo-wide // namespace restriction above, raw `node:fs`/`node:sqlite` are confined to namespace drivers - // and a short enumerated list of existing files. The files below are pre-existing direct - // namespace consumers the facade doesn't cover yet (tracked for step 3) or raw-primitive - // users; they get their own, narrower overrides further down. + // and a short enumerated list of files. The excluded files are direct namespace consumers + // the facade doesn't cover or raw-primitive users; they get narrower overrides further down. + // oxlint JSON has no shared lists, so the direct-consumer files are listed here and again in + // the override that restores the raw-primitive rules for them; keep the two in sync. "files": ["packages/stack/src/**", "packages/stack/tests/**"], "excludeFiles": [ "packages/stack/src/namespace/**", @@ -243,8 +244,8 @@ } }, { - // Existing direct namespace consumers the facade doesn't cover yet (tracked for step 3); - // none of them touch raw node:fs/node:sqlite themselves, so that restriction stays on. + // Direct namespace consumers the facade doesn't cover; none of them touch raw + // node:fs/node:sqlite themselves, so that restriction stays on. // - Network.ts, Ports.ts (+its integration test), port-reservation.e2e.test.ts: read/reserve // ports through PortReservations, not yet on the facade. // - Owner.ts: needs PortReservations and owned/borrowed Paths. @@ -304,8 +305,8 @@ }, { // `host/CommandRunner.ts` needs Environment and borrow from the namespace, plus the - // raw `rmdir` Effect's FileSystem has no non-recursive equivalent for (see its existing - // disable comment); sqlite stays off-limits. + // raw `rmdir` Effect's FileSystem has no non-recursive equivalent for; sqlite stays + // off-limits. "files": ["packages/stack/src/host/CommandRunner.ts"], "rules": { "no-restricted-imports": [ @@ -352,16 +353,16 @@ }, { // Standalone process outside the Effect runtime, launched before any service is wired up: it - // calls the sqlite-pin driver and raw node:fs/node:sqlite directly by design (see its - // existing disable comments), so none of the namespace restrictions apply here. + // calls the sqlite-pin driver and raw node:fs/node:sqlite directly by design, so none of + // the namespace restrictions apply here. "files": ["packages/stack/src/runtime/native-launcher.ts"], "rules": { "no-restricted-imports": "off" } }, { - // Raw node:fs with an existing disable comment explaining why Effect's FileSystem can't do - // the job; none of these import namespace internals or raw sqlite. + // Raw node:fs where Effect's FileSystem can't do the job (each import's disable comment + // gives the reason); none of these import namespace internals or raw sqlite. "files": [ "packages/stack/src/internal/host-process.ts", "packages/stack/src/Sweep.integration.test.ts", @@ -406,8 +407,8 @@ }, { // Standalone fixture process the lease waits on, outside any Effect runtime: it takes the - // same raw sqlite lock as the lease it waits for and reports over a raw fd (see its existing - // disable comments). It doesn't import namespace internals, so that restriction stays on. + // same raw sqlite lock as the lease it waits for and reports over a raw fd. + // It doesn't import namespace internals, so that restriction stays on. "files": ["packages/stack/tests/lease-wait-fixture.ts"], "rules": { "no-restricted-imports": [ diff --git a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md index 0bd3d3d165..71d98af30e 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md @@ -20,12 +20,12 @@ does not launch an owner. After confirmation the command opens the selected handle and destroys its entire namespace. Destruction may start an owner to clean up a stopped namespace. Cleanup failures remain errors; the command does not claim success on failure. -A failed destroy may leave its owner running; retry destruction or use -`stack stop` to shut down that owner. +Shutdown is one-way: a failed destroy leaves the stack registered and its +owner exits; the next command retries. -As with `stack stop`, the owner drains live connections on every stack listener -(up to 10 seconds) before any service stops, so a request in flight when -destroy is issued can still complete. +As with `stack stop`, services stop in reverse dependency order with every +stack listener still open, and traffic that would wake a stopped service during +shutdown is refused. When no owner is running and the engine reports that its daemon cannot be reached, destruction removes nothing: the registration, port claims and host @@ -62,7 +62,7 @@ remains the backstop when no owner is left alive to do so. ## Output, exit codes and telemetry Text prints `Stack destroyed.`, or, when engine cleanup was skipped, -`Stack could not be fully destroyed because Docker is unreachable; restore it and run "supabase stack destroy" again.` +`Stack could not be fully destroyed because Docker is unreachable; restore it and run "supabase stack destroy --stack-id " again.` JSON and stream-json success data contain `destroyed` (`false` when engine cleanup was skipped), `id`, and `runtimeCleanup` (`complete` or `skipped`); a skipped cleanup also carries `engine`. Exit 0 on destruction or skipped engine cleanup, 1 on invalid flags, missing diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index d0277d3aa5..c7fdc361d4 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -69,9 +69,9 @@ stay owner-restricted between launches otherwise. A directory inside the cached never chmodded; it keeps the archive's own mode, and startup fails if one is not already traversable. Database is eager by default. Other services are lazy; traffic wakes them through their listeners. -Lazy services with idle policies stop after 60 seconds without traffic, Studio after 5 minutes. A +Lazy services stop after 60 seconds without traffic, Studio after 5 minutes. A service that a running service depends on, such as pg-meta for Studio, stays up until that -dependent stops. Functions has no automatic idle stop. `--eager` makes all selected services eager. +dependent stops. `--eager` makes all selected services eager. Changes to activation policy take effect after stopping and starting the stack, including when a later invocation omits an earlier `--eager` flag. `--preparation` selects on-demand or background artifact preparation. diff --git a/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md index 02418df95c..aff47bcbc3 100644 --- a/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md @@ -18,11 +18,11 @@ It fails when another process holds the lease or the cleanup fails, so a stack whose owner exited after a failed stop can be stopped again. If an owner disappears after discovery, the shutdown error remains a failure. -Before any service stops, the owner stops accepting new connections on every -stack listener and lets connections already established keep flowing for up to -10 seconds, then cuts whatever remains. A request in flight when `stop` is -issued can still complete, at the cost of the command waiting up to that long -longer for shutdown to finish. +Services stop in reverse dependency order with every stack listener still open, +so a dependent's graceful stop can still reach its prerequisite; each listener +closes once its own service has stopped. Traffic that would wake a stopped +service during shutdown is refused. Shutdown is one-way: a failed stop leaves +the stack registered and the owner exits, and the next stop retries. Text confirms each successful shutdown and identifies each stack that had no owner. JSON and stream-json success data contain `stopped` and `notRunning` ID arrays. diff --git a/docs/adr/0017-simplified-managed-stack-architecture.md b/docs/adr/0017-simplified-managed-stack-architecture.md index 234243f38f..b02dac32e0 100644 --- a/docs/adr/0017-simplified-managed-stack-architecture.md +++ b/docs/adr/0017-simplified-managed-stack-architecture.md @@ -137,11 +137,10 @@ it selects Docker when the daemon is reachable and native otherwise. An installed client with an unreachable daemon selects native, and the created Effect handle carries a notice that the selection is persisted for that stack, so a later switch to Docker requires destroying the stack or choosing a new -stack name; callers such as `stack start` decide whether to print it. Native is -refused as uid 0. Existing state is reused without probing. Callers may -explicitly select native, Docker, or Podman without fallback, and an omitted -engine for an explicit container runtime defaults to Docker. Podman is -supported only on local Linux hosts. See +stack name; callers such as `stack start` decide whether to print it. Native as +root runs only PostgreSQL as an unprivileged user. Existing state is reused +without probing. Callers may explicitly select native or Docker without +fallback. See [ADR-0025](0025-ephemeral-postgres-for-schema-tooling.md) for the daemon probe and its interaction with ephemeral Postgres. Persisted state records the resolved exact engine. Capability releases and diff --git a/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md b/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md index 0b55385e27..b65f561104 100644 --- a/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md +++ b/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md @@ -129,8 +129,10 @@ daemon persists Docker. A present client with a dead daemon persists **native** notice that destroy-and-recreate (or a new `--stack` name) is required to get Docker later. Persisted runtime never flips. Explicit `--runtime docker` still requires a live daemon. -Native Postgres is refused when the process uid is 0 (`initdb` refuses root). There is no -uid-drop. Use `--runtime docker`. +Native Postgres refuses to run as uid 0 (`initdb` refuses root), so when the stack runs as root +only the PostgreSQL process steps down to an unprivileged user: one named by +`SUPABASE_NATIVE_POSTGRES_USER`, or the account of a detected agent sandbox. Otherwise use +`--runtime docker`. ### Optional catalog downloads @@ -149,7 +151,7 @@ leaving schema policy in the CLI. ### Positive -- Native and Docker/Podman shadows share one API and the same slim baseline as `stack start`. +- Native and Docker shadows share one API and the same slim baseline as `stack start`. - Schema commands can target a running project stack through `credentials()` when the flag is on. `credentials().database` is available whenever the database listener is assigned, including when Auth is disabled. `credentials().api` is absent when Auth is off. Overlay and `--local` keep @@ -166,7 +168,7 @@ leaving schema policy in the CLI. - A failed first live setup after the engine is running is stuck until `db reset`, same as Compose. A failed cold launch that never reached running retries first-create. - Windows native dump/test/squash need a working Docker client even though Postgres itself is native. -- Native stacks as uid 0 cannot start; Docker (or a non-root user) is required. +- Native stacks as uid 0 need `SUPABASE_NATIVE_POSTGRES_USER` or a detected agent sandbox; otherwise Docker (or a non-root user) is required. - Auto-selected native after a dead Docker daemon is sticky until destroy or a new stack name. ## Alternatives Considered diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 1abdd4acce..e11f041968 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -273,7 +273,7 @@ Admission requires the target and its whole prerequisite closure to be ready. An ### Crash-loop breaker -Each launch failure or unrequested exit counts once per generation. Three consecutive failures open the breaker for a fixed 30 seconds. While open, admission fails at once naming the last cause; the cooldown's end re-admits one shared recovery attempt, and a further failure reopens it for the same 30 seconds. An explicit start, restart or composition start resets the breaker. +Each launch failure or unrequested exit counts once per generation. Three failures, each within five minutes of the previous one, open the breaker for a fixed 30 seconds; a failure after a longer quiet spell starts a new count. While open, admission fails at once naming the last cause; the cooldown's end re-admits one shared recovery attempt, and a further failure reopens it for the same 30 seconds. An explicit start, restart or composition start resets the breaker. ### Chart 3: dependency awareness lives in the reducer @@ -496,15 +496,14 @@ interval (30 seconds by default) and treats only a confirmed ENOENT as gone; any such as a permission failure or a transient I/O error on an unmounted root, keeps the owner running and logs a warning instead. On a confirmed-gone registration the owner treats its ownership as ended: it stops every workload (ordered stop, above) and removes what it -created through the same registration-independent `cleanupResources` path destroy uses, retrying -transient cleanup failures and giving up only once an engine is confirmed unreachable (the leftover -container ids are logged, not retried forever), releases its port reservations, and exits. It never -reads or writes the registration again, and it does not publish a registration removal: there is -nothing left to update, and any resource claim it already recorded stays available for a future -reconcile. Deletion, a move and an unmount are treated identically, because a stack whose root is +created through the same registration-independent instance cleanup destroy uses, retrying +transient cleanup failures and giving up only once an engine is confirmed unreachable (what it +could not remove is logged, not retried forever), and exits. It never reads or writes the +registration again, and it does not release its port reservations: a stale reservation is +reclaimed lazily by whichever stack next needs the port. Deletion, a move and an unmount are treated identically, because a stack whose root is unreachable cannot operate regardless of which of the three caused it. -**Unreachable engine.** An owner of a container stack fails startup with a `runtime-unavailable` reason when its first container sweep finds the engine CLI missing or its daemon not listening; permission, TLS, authentication and timeout failures are ordinary startup errors. `destroy` then proceeds without an owner: it takes the free lease, publishing a sweeper record like the orphan sweep, refuses when any stack data directory cannot be deleted by the current user, removes the host data, the registration and its port reservations, and returns the shell commands that remove the stack's containers and engine-volume data once the engine runs. `stop` without a live owner takes the same lease and reclaims the registered stack's leftovers (the orphan-sweep path: containers, plus destruction of a session stack) before succeeding; it fails if another process holds the lease or the cleanup fails, and so needs the engine for a container stack. +**Unreachable engine.** An owner of a container stack fails startup with a `runtime-unavailable` reason when its first container sweep finds the engine CLI missing or its daemon not listening; permission, TLS, authentication and timeout failures are ordinary startup errors. `destroy` then proceeds without an owner: it only confirms that no owner holds the stack's lease, leaves the registration, port reservations and host data untouched, and returns `{ runtimeCleanup: "skipped", engine }`; run it again once the engine is reachable. `stop` without a live owner takes the same lease and reclaims the registered stack's leftovers (the orphan-sweep path: containers, plus destruction of a session stack) before succeeding; it fails if another process holds the lease or the cleanup fails, and so needs the engine for a container stack. During Serving, keep the owner alive independently of callers. Sleeping instances still need its public listeners. This is process lifetime management, not automatic service restart or continuous reconciliation. diff --git a/packages/stack/README.md b/packages/stack/README.md index 8c32e93d06..a1ffd65e39 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -177,11 +177,11 @@ await stack.composition.configure({ await stack.composition.start(); ``` -Bindings supply ordinary configuration values; a URL alone never creates a dependency. An independent REST instance can instead receive an external database URL. Lazy members receive public listeners before their processes start. Public traffic starts an armed instance and waits for health, up to a 120-second wake budget; inspector connections can attach before health succeeds. A prerequisite stays awake while any dependent runs. A crashed instance stays armed and the next request wakes it again; after three consecutive failures it fails fast for a cooldown of 30 seconds, doubling up to 5 minutes, until the cooldown elapses or an explicit start. Explicit stop disables wake. An individual restart runs the instance explicitly; restart the composition to reapply its lazy policy. +Bindings supply ordinary configuration values; a URL alone never creates a dependency. An independent REST instance can instead receive an external database URL. Lazy members receive public listeners before their processes start. Public traffic starts an armed instance and waits for health, up to a 120-second wake budget; inspector connections can attach before health succeeds. A prerequisite stays awake while any dependent runs. A crashed instance stays armed and the next request wakes it again; after repeated failures it fails fast for a 30-second cooldown, until the cooldown elapses or an explicit start. Explicit stop disables wake. An individual restart runs the instance explicitly; restart the composition to reapply its lazy policy. - Instance methods affect that instance, subject to dependency checks. - Composition methods affect selected members; startup also includes declared prerequisites. -- `stack.stop()` stops every owned instance and attached command and returns after confirming owner exit. Without a live owner nothing runs, so it returns without starting one; an instance's `stop()` behaves the same way. +- `stack.stop()` stops every owned instance and attached command and returns after confirming owner exit. Without a live owner it does not start one: it reclaims the stack's leftover workloads under the stack lease, and fails if another process holds the lease or the cleanup fails. An instance's `stop()` without a live owner has nothing to stop and returns without starting one. - `stack.destroy()` additionally removes owned data and registrations, and also waits for owner exit. - `stack.close()` disposes the client and invalidates its active observation iterators. Closing the creating client of a session stack destroys the stack. Stopping the last instance leaves the owner available. diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index eef6c8b680..d85c958bf1 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -152,9 +152,6 @@ const runRaw = ( * (`--host `), since the docker CLI gives that priority over `DOCKER_HOST` and context * switches, and since an argument, unlike an environment variable, never leaks to subprocesses a * launched workload spawns. - * - * `engine` keeps the seam a future Podman driver would plug into; only `"docker"` is resolved - * today. */ export interface EngineTarget { readonly engine: "docker"; From 658e5e97c87e4635259420c92e95e5b0eba862ca Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 11:19:11 +0200 Subject: [PATCH 082/142] fix(stack): remove everything a destroyed stack left under its data root --- packages/stack/src/Owner.integration.test.ts | 30 ++++++++++++++++++++ packages/stack/src/Owner.ts | 19 +++++++++++-- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/packages/stack/src/Owner.integration.test.ts b/packages/stack/src/Owner.integration.test.ts index 33e8fe30e8..0f2ac45c51 100644 --- a/packages/stack/src/Owner.integration.test.ts +++ b/packages/stack/src/Owner.integration.test.ts @@ -863,6 +863,36 @@ it.live("destroying one service deletes its port reservation", () => ), ); +it.live("destroying a stack leaves its data root empty after an owner was killed mid-command", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-orphaned-job-" }); + const stack = initial(`owner-orphaned-job-${randomUUID().slice(0, 8)}`); + const state = yield* stateFor(`${root}/state`); + yield* state.save(stack); + const owner = yield* ownerFor({ saved: stack, state, root: `${root}/data`, cacheRoot }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + const orphanedJob = `${root}/data/jobs/0b1c2d3e-orphan`; + yield* fs.makeDirectory(orphanedJob, { recursive: true }); + yield* fs.writeFileString(`${orphanedJob}/config.json`, "{}"); + + yield* owner.namespace.destroy; + + expect(yield* fs.exists(`${root}/data`)).toBe(true); + expect(yield* fs.readDirectory(`${root}/data`)).toEqual([]); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), +); + it.effect("refuses to generate credentials for a stack whose saved instances consume them", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index d9120f680d..b83ca01e98 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -275,11 +275,24 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ? Effect.succeed(true) : Container.probeEngineReachable(spawner, options.engineTarget); // Shared per-launch environment-file scratch directory (Container.ts), owned by this stack as a - // whole rather than any one instance; registration-independent so destroy and abandonment both - // reach it without depending on `Registry.remove`, which never runs during abandonment. + // whole rather than any one instance; registration-independent so abandonment reaches it + // without `Registry.remove`, which never runs then. const removeContainerEnvRoot = fs .remove(path.join(options.root, Paths.CONTAINER_ENV_DIRNAME), { recursive: true, force: true }) .pipe(Effect.mapError(serviceError("cleanup"))); + // Everything under the data root belongs to the stack, including what a killed owner left + // mid-command (`jobs/`); the directory itself stays for `Registry.pruneDestroyed`. + const removeDataRootEntries = fs.exists(options.root).pipe( + Effect.flatMap((exists) => (exists ? fs.readDirectory(options.root) : Effect.succeed([]))), + Effect.flatMap((names) => + Effect.forEach( + names, + (name) => fs.remove(path.join(options.root, name), { recursive: true, force: true }), + { discard: true }, + ), + ), + Effect.mapError(serviceError("cleanup")), + ); const rejectWhileDraining = (options.draining ?? Effect.succeed(false)).pipe( Effect.flatMap((isDraining) => isDraining @@ -796,7 +809,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { // that is best-effort because `isGone` reclaims a gone stack's rows lazily. destroy: orchestrator.destroyNamespace.pipe( Effect.andThen(sweep), - Effect.andThen(removeContainerEnvRoot), + Effect.andThen(removeDataRootEntries), Effect.andThen(options.state.remove(stackId)), Effect.andThen( network.releaseStack.pipe( From e58ebb1901780ee22d4be2d7df9073b2c36819c0 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 11:21:27 +0200 Subject: [PATCH 083/142] fix(stack): let an instance destroy remove the service despite open leases --- packages/stack/ARCHITECTURE.md | 2 ++ packages/stack/src/Lifecycle.ts | 12 +++++--- .../src/Orchestrator.integration.test.ts | 29 +++++++++++++++++++ packages/stack/src/Orchestrator.ts | 11 ++++--- 4 files changed, 46 insertions(+), 8 deletions(-) diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index e11f041968..43bce6e1ee 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -271,6 +271,8 @@ A service has demand while it holds leases or waiters, after an explicit start, Admission requires the target and its whole prerequisite closure to be ready. An inspector acquisition needs only a live target session. A readiness failure is not terminal: the session keeps running, and the waiters that awaited that readiness itself, or explicitly through a dependent, fail with its error. While any other waiter is still blocked on it, the reducer keeps re-checking that generation, one check at a time and one second apart after a failed re-check, until it recovers, its generation ends or no waiter needs it; the waiters' budgets bound how long that lasts. Losing a prerequisite never replaces a healthy dependent; a crashed prerequisite with running dependents relaunches because they keep demand on it. +A destroy that holds its storage reservation owns the service: removing it from the graph succeeds even while work leases are outstanding, and a lease released afterwards is ignored. + ### Crash-loop breaker Each launch failure or unrequested exit counts once per generation. Three failures, each within five minutes of the previous one, open the breaker for a fixed 30 seconds; a failure after a longer quiet spell starts a new count. While open, admission fails at once naming the last cause; the cooldown's end re-admits one shared recovery attempt, and a further failure reopens it for the same 30 seconds. An explicit start, restart or composition start resets the breaker. diff --git a/packages/stack/src/Lifecycle.ts b/packages/stack/src/Lifecycle.ts index 81ce08deeb..1d8c05460c 100644 --- a/packages/stack/src/Lifecycle.ts +++ b/packages/stack/src/Lifecycle.ts @@ -1023,10 +1023,14 @@ const applyEvent = ( !specsEqual(previousSpec, nextSpec) ); }); - const blockedIds = [...removedIds, ...changedIds].filter((id) => { - const service = state.services.get(id); - return service !== undefined && isActiveForGraphUpdate(service); - }); + // A destroy holding its reservation owns the service, so its removal can't be blocked. + const destroyed = (id: string) => state.services.get(id)?.destroy === "reserved"; + const blockedIds = [...removedIds.filter((id) => !destroyed(id)), ...changedIds].filter( + (id) => { + const service = state.services.get(id); + return service !== undefined && isActiveForGraphUpdate(service); + }, + ); if (blockedIds.length > 0) { for (const id of blockedIds) commands.push( diff --git a/packages/stack/src/Orchestrator.integration.test.ts b/packages/stack/src/Orchestrator.integration.test.ts index b5b1b57722..852aceab20 100644 --- a/packages/stack/src/Orchestrator.integration.test.ts +++ b/packages/stack/src/Orchestrator.integration.test.ts @@ -747,6 +747,35 @@ describe("service composition", () => { ); }); +describe("destroying an instance with admitted traffic", () => { + it.live("removes the instance even while its traffic lease is held", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + yield* makeInstance(orchestrator, "database"); + yield* makeInstance(orchestrator, "rest"); + yield* orchestrator.configure({ + members: [ + { id: "database", activation: "lazy" }, + { id: "rest", activation: "lazy" }, + ], + dependencies: [{ from: "database", to: "rest" }], + }); + yield* orchestrator.startComposition; + const request = yield* Scope.make(); + yield* orchestrator.acquire("rest").pipe(Scope.provide(request)); + + yield* orchestrator.destroy("rest"); + expect((yield* orchestrator.instances).map(({ id }) => id)).not.toContain("rest"); + yield* orchestrator.stop("database"); + + yield* Scope.close(request, Exit.void); + expect((yield* orchestrator.status("database")).lifecycle).toBe("stopped"); + }), + ), + ); +}); + describe("wake and idle sleep", () => { it.live( "does not let a prerequisite idle-sleep while its dependent is still preparing to wake", diff --git a/packages/stack/src/Orchestrator.ts b/packages/stack/src/Orchestrator.ts index d6ec25ff61..56d1f9f2fb 100644 --- a/packages/stack/src/Orchestrator.ts +++ b/packages/stack/src/Orchestrator.ts @@ -946,7 +946,6 @@ export const make = Effect.fn("Orchestrator.make")(function* member.id !== id), @@ -954,11 +953,15 @@ export const make = Effect.fn("Orchestrator.make")(function* dependency.from !== id && dependency.to !== id, ), }; - yield* Ref.set(composition, next); - yield* applyLocked([ - LifecycleEvent.DestroyReleased({ id }), + // The graph update goes first: it removes the service while the destroy still holds it. + const applied = yield* applyLocked([ LifecycleEvent.GraphUpdated({ graph: lifecycleGraph(values.keys(), next) }), + LifecycleEvent.DestroyReleased({ id }), ]); + const rejected = rejectionFor(applied.commands, id); + if (rejected !== undefined) return yield* rejected; + yield* Ref.set(registry, values); + yield* Ref.set(composition, next); }), ); From f27779e08303628078d3162b25dfcea63dfaf7ed Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 11:46:10 +0200 Subject: [PATCH 084/142] fix(stack): prune destroyed stack directories under the registry lock --- .../namespace/Registry.integration.test.ts | 53 +++++++++++++++++++ packages/stack/src/namespace/Registry.ts | 50 +++++++++-------- 2 files changed, 82 insertions(+), 21 deletions(-) create mode 100644 packages/stack/src/namespace/Registry.integration.test.ts diff --git a/packages/stack/src/namespace/Registry.integration.test.ts b/packages/stack/src/namespace/Registry.integration.test.ts new file mode 100644 index 0000000000..2afdec4030 --- /dev/null +++ b/packages/stack/src/namespace/Registry.integration.test.ts @@ -0,0 +1,53 @@ +import { NodeServices } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { Context, Effect, Fiber, FileSystem, Layer, Path } from "effect"; +import { TestClock } from "effect/testing"; +import * as StackNamespace from "../StackNamespace.ts"; + +const RETENTION_SECONDS = 11 * 60; + +it.effect( + "pruning a destroyed shell cannot remove directories while its id is being registered", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-registry-prune-" }); + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root })), + StackNamespace.Service, + ); + const id = "recreated"; + const directory = path.join(root, id); + yield* fs.makeDirectory(path.join(directory, "data"), { recursive: true }); + const destroyedAtSeconds = 1_767_225_600; + yield* fs.utimes(directory, destroyedAtSeconds, destroyedAtSeconds); + yield* TestClock.setTime((destroyedAtSeconds + RETENTION_SECONDS) * 1000); + + const scope = yield* Effect.scope; + yield* state.withLock( + Effect.gen(function* () { + const pruning = yield* state.pruneDestroyed("other").pipe(Effect.forkIn(scope)); + // The controlled clock moves the prune's lock retries along until it finishes, whether + // by pruning or by giving up on the busy lock. + yield* Fiber.await(pruning).pipe( + Effect.race(TestClock.adjust("50 millis").pipe(Effect.forever)), + ); + expect(yield* fs.exists(path.join(directory, "data"))).toBe(true); + yield* state.save({ + id, + runtime: "native", + identity: { projectRoot: root, branchContext: "test", stackName: id }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + }); + }), + ); + + expect(yield* fs.exists(path.join(directory, "data"))).toBe(true); + expect((yield* state.read(id))?.id).toBe(id); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/packages/stack/src/namespace/Registry.ts b/packages/stack/src/namespace/Registry.ts index 9346ab8933..c2e1e9f2d5 100644 --- a/packages/stack/src/namespace/Registry.ts +++ b/packages/stack/src/namespace/Registry.ts @@ -204,27 +204,6 @@ export const make = ( const modified = Option.getOrUndefined((yield* fs.stat(directory)).mtime)?.getTime(); return modified !== undefined && now - modified >= DESTROYED_RETENTION_MILLIS; }); - const pruneDestroyed = Effect.fn("Namespace.Registry.pruneDestroyed")(function* (keep: string) { - const now = yield* Clock.currentTimeMillis; - let removed = 0; - for (const id of yield* stackIds) { - if (id === keep) continue; - const pruned = yield* Effect.gen(function* () { - if (!(yield* isDestroyedShell(id, now))) return false; - yield* removeEmptyDirectory(path.join(stackRoot(id), "data")); - yield* removeEmptyDirectory(stackRoot(id)); - return true; - }).pipe( - Effect.catch((error) => - Effect.logWarning(`Could not prune destroyed stack directory ${id}`, error).pipe( - Effect.as(false), - ), - ), - ); - if (pruned) removed += 1; - } - yield* Effect.annotateCurrentSpan({ removed }); - }); /** * One attempt: forks `attemptScope` under `guardScope` before opening anything, so there is * always a scope to close if the open or the `BEGIN IMMEDIATE` below fails or defects. Runs @@ -275,5 +254,34 @@ export const make = ( ), ); + const pruneDestroyed = Effect.fn("Namespace.Registry.pruneDestroyed")(function* (keep: string) { + const now = yield* Clock.currentTimeMillis; + let removed = 0; + for (const id of yield* stackIds) { + if (id === keep) continue; + // Rechecked under the registry lock: a same-id recreate either saved its state before the + // check or waits until the directories are gone. + const pruned = yield* Effect.gen(function* () { + if (!(yield* isDestroyedShell(id, now))) return false; + return yield* withLock( + Effect.gen(function* () { + if (!(yield* isDestroyedShell(id, now))) return false; + yield* removeEmptyDirectory(path.join(stackRoot(id), "data")); + yield* removeEmptyDirectory(stackRoot(id)); + return true; + }), + ); + }).pipe( + Effect.catch((error) => + Effect.logWarning(`Could not prune destroyed stack directory ${id}`, error).pipe( + Effect.as(false), + ), + ), + ); + if (pruned) removed += 1; + } + yield* Effect.annotateCurrentSpan({ removed }); + }); + return { read, list: list(), save, remove, pruneDestroyed, withLock }; }); From 4b40935dff36005c8412853c0d238d769e2d57b1 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 11:46:13 +0200 Subject: [PATCH 085/142] test(stack): keep port collision tests independent of foreign listeners and leftover entries --- packages/stack/src/Ports.integration.test.ts | 23 +++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/packages/stack/src/Ports.integration.test.ts b/packages/stack/src/Ports.integration.test.ts index a3c1036be4..9fc71f2586 100644 --- a/packages/stack/src/Ports.integration.test.ts +++ b/packages/stack/src/Ports.integration.test.ts @@ -118,20 +118,29 @@ it.live( // there leaves no gap for a parallel test to take the port before the foreign listener. const foreign = Net.createServer(); let firstCandidate: number | undefined; + let foreignBound = false; const bind = (host: string, port: number) => Effect.gen(function* () { if (firstCandidate === undefined) { firstCandidate = port; + // EADDRINUSE means an unrelated process already occupies the candidate, which is the + // collision this test needs. yield* Effect.callback((resume) => { - foreign.once("error", (cause) => resume(Effect.die(cause))); - foreign.listen(port, "127.0.0.1", () => resume(Effect.void)); + foreign.once("error", (cause: NodeJS.ErrnoException) => + resume(cause.code === "EADDRINUSE" ? Effect.void : Effect.die(cause)), + ); + foreign.listen(port, "127.0.0.1", () => { + foreignBound = true; + resume(Effect.void); + }); }); } return yield* probed(host, port); }); yield* Effect.addFinalizer(() => Effect.callback((resume) => { - foreign.close(() => resume(Effect.void)); + if (foreignBound) foreign.close(() => resume(Effect.void)); + else resume(Effect.void); }), ); @@ -260,9 +269,10 @@ it.live( }; yield* state.save(saved); - // Fails removing the stack's scratch directory on the first attempt only, standing in for - // any concrete cleanup failure (an unreachable engine, a locked file, and so on); resolved - // for the retry once the uncertainty would realistically have cleared. + // Fails removing a leftover scratch directory under the data root on the first attempt + // only, standing in for any concrete cleanup failure (an unreachable engine, a locked + // file, and so on); resolved for the retry once the uncertainty would realistically have + // cleared. const scratch = path.join(root, "data", CONTAINER_ENV_DIRNAME); let resolved = false; const unreliableFileSystem = Layer.effect( @@ -315,6 +325,7 @@ it.live( ); expect(yield* portReservations.find(realStateRoot, id, `${mail.id}:http`)).toBeDefined(); + yield* fs.makeDirectory(scratch, { recursive: true }); const firstAttempt = yield* owner.namespace.destroy.pipe(Effect.exit); expect(firstAttempt._tag).toBe("Failure"); // Retained: the row survives a destroy that could not confirm nothing remains. From d2cac322b5786770737964a2312feda212e376ae Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 12:21:25 +0200 Subject: [PATCH 086/142] test(stack): observe the database stop through a live docker events stream --- .../src/services/Database.integration.test.ts | 50 ++++++++++++------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/packages/stack/src/services/Database.integration.test.ts b/packages/stack/src/services/Database.integration.test.ts index 7e2ba6dd87..31bfdc7f5e 100644 --- a/packages/stack/src/services/Database.integration.test.ts +++ b/packages/stack/src/services/Database.integration.test.ts @@ -5,6 +5,7 @@ import { Context, Deferred, Effect, + Fiber, FileSystem, Layer, Path, @@ -13,6 +14,7 @@ import { Ref, Stream, } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { createHash } from "node:crypto"; import { tmpdir } from "node:os"; import { DEFAULT_POSTGRES_ROOT_KEY } from "../Defaults.ts"; @@ -601,6 +603,36 @@ describe("database component", { timeout: 180_000 }, () => { container, ]); + // A bounded `docker events --until` replay intermittently returns nothing on Docker Desktop, + // so subscribe live before stop; `--since` replays anything logged before the stream attaches. + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const events = yield* spawner.spawn( + ChildProcess.make( + "docker", + [ + "events", + "--since", + startedAt.output.trim(), + "--filter", + `container=${container}`, + "--filter", + "event=kill", + "--filter", + "event=die", + "--format", + '{{.Action}} {{index .Actor.Attributes "signal"}}{{index .Actor.Attributes "exitCode"}}', + ], + { stdin: "ignore", stdout: "pipe", stderr: "ignore" }, + ), + ); + const observed = yield* events.stdout.pipe( + Stream.decodeText, + Stream.splitLines, + Stream.takeUntil((line) => line.startsWith("die ")), + Stream.runCollect, + Effect.forkScoped, + ); + yield* Effect.scoped( Effect.gen(function* () { const services = yield* Layer.build( @@ -620,24 +652,8 @@ describe("database component", { timeout: 180_000 }, () => { }), ); - const stoppedAt = yield* runDocker(["info", "--format", "{{.SystemTime}}"]); - const events = yield* runDocker([ - "events", - "--since", - startedAt.output.trim(), - "--until", - stoppedAt.output.trim(), - "--filter", - `container=${container}`, - "--filter", - "event=kill", - "--filter", - "event=die", - "--format", - '{{.Action}} {{index .Actor.Attributes "signal"}}{{index .Actor.Attributes "exitCode"}}', - ]); expect( - events.output.trim().split("\n"), + yield* Fiber.join(observed), "stop sends SIGINT and PostgreSQL exits cleanly", ).toEqual(["kill 2", "die 0"]); yield* service.destroy; From 9b6a3f5d467db1eb2f573d82a7ba0a28645c07d5 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 12:27:17 +0200 Subject: [PATCH 087/142] test(stack): report a foreign-held port candidate as a collision --- packages/stack/src/Ports.integration.test.ts | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/packages/stack/src/Ports.integration.test.ts b/packages/stack/src/Ports.integration.test.ts index 9fc71f2586..b8e6b53324 100644 --- a/packages/stack/src/Ports.integration.test.ts +++ b/packages/stack/src/Ports.integration.test.ts @@ -123,17 +123,20 @@ it.live( Effect.gen(function* () { if (firstCandidate === undefined) { firstCandidate = port; - // EADDRINUSE means an unrelated process already occupies the candidate, which is the - // collision this test needs. - yield* Effect.callback((resume) => { + foreignBound = yield* Effect.callback((resume) => { foreign.once("error", (cause: NodeJS.ErrnoException) => - resume(cause.code === "EADDRINUSE" ? Effect.void : Effect.die(cause)), + resume(cause.code === "EADDRINUSE" ? Effect.succeed(false) : Effect.die(cause)), ); - foreign.listen(port, "127.0.0.1", () => { - foreignBound = true; - resume(Effect.void); - }); + foreign.listen(port, "127.0.0.1", () => resume(Effect.succeed(true))); }); + // An unrelated process already holds the candidate: report the collision it caused + // rather than probing a listener this test doesn't own. + if (!foreignBound) + return yield* new PortError({ + key: "api", + message: `Port ${port} is already in use`, + conflict: { port, endpoint: "api", holder: "foreign" }, + }); } return yield* probed(host, port); }); From 9e993bb56216011e33625941c0992aeb9a00b826 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 13:05:57 +0200 Subject: [PATCH 088/142] feat(stack): run container stacks on Podman through the pinned engine target --- packages/stack/src/Network.ts | 2 +- packages/stack/src/StackHost.ts | 4 +- packages/stack/src/Sweep.ts | 7 +- packages/stack/src/effect.ts | 6 +- packages/stack/src/host/CommandRunner.ts | 4 +- packages/stack/src/namespace/Registry.ts | 2 +- packages/stack/src/runtime/Container.ts | 135 +++++++++++++++--- packages/stack/src/services/Database.ts | 4 +- .../stack/src/services/DatabaseSnapshot.ts | 2 +- packages/stack/src/services/Recipe.ts | 2 +- .../src/storage/DockerDatabaseStorage.ts | 44 +++++- packages/stack/src/testing.ts | 2 +- 12 files changed, 175 insertions(+), 39 deletions(-) diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index 8dfe45ad8f..7318b244b2 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -6,7 +6,7 @@ import { makePorts, PortError, probeVacant } from "./Ports.ts"; import { bindTcp, serveTcp, type BackendAddress, type ProxyError } from "./Proxy.ts"; import { makeHttpProxy, type HttpProxy, type HttpRoute } from "./HttpProxy.ts"; -export type NetworkRuntime = "native" | "docker"; +export type NetworkRuntime = "native" | "docker" | "podman"; type RouteContribution = Pick< HttpRoute, diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index 6efbabfe47..c335d391b5 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -99,7 +99,7 @@ const engineUnreachableAtStartup = (cause: unknown): boolean => { cause.cause instanceof PlatformError.PlatformError && cause.cause.reason._tag === "NotFound" && cause.cause.reason.method === "spawn") || - /cannot connect to the docker daemon|connection refused|connect: no such file or directory|error during connect:[^\n]*(?:docker daemon is not running|the system cannot find the file specified)/iu.test( + /cannot connect to the docker daemon|connection refused|connect: no such file or directory|error during connect:[^\n]*(?:docker daemon is not running|the system cannot find the file specified)|unable to connect to podman socket:[^\n]*the system cannot find the file specified/iu.test( message, ) ); @@ -552,7 +552,7 @@ export const runStackHost = Effect.fn("StackHost.run")( const engineTarget = saved.runtime === "native" ? undefined - : yield* resolveEngineTarget(spawner).pipe( + : yield* resolveEngineTarget(spawner, saved.runtime).pipe( Effect.mapError((cause) => hostError( "startup-cleanup", diff --git a/packages/stack/src/Sweep.ts b/packages/stack/src/Sweep.ts index 4737524447..bad3d17e0f 100644 --- a/packages/stack/src/Sweep.ts +++ b/packages/stack/src/Sweep.ts @@ -13,7 +13,8 @@ const destroyStack = Effect.fn("Sweep.destroyStack")(function* ( cacheRoot: string, ) { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const engineTarget = saved.runtime === "native" ? undefined : yield* resolveEngineTarget(spawner); + const engineTarget = + saved.runtime === "native" ? undefined : yield* resolveEngineTarget(spawner, saved.runtime); const context = yield* Layer.build( Owner.layer({ saved, root: dataRoot, cacheRoot, engineTarget }).pipe( Layer.provide(Layer.succeed(StackNamespace.Service, state)), @@ -55,7 +56,9 @@ export const reclaimStack = Effect.fn("Sweep.reclaimStack")(function* (options: else { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const engineTarget = - saved.runtime === "native" ? undefined : yield* resolveEngineTarget(spawner); + saved.runtime === "native" + ? undefined + : yield* resolveEngineTarget(spawner, saved.runtime); yield* Owner.sweepContainers(saved, dataRoot, engineTarget); } return true; diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index 0fbfa54819..dd8a41ca9e 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -94,7 +94,7 @@ export interface StackLocations { export interface CreateOptions extends StackLocations { readonly projectRoot: string; readonly name?: string; - readonly runtime: "native" | "docker"; + readonly runtime: "native" | "docker" | "podman"; /** * A `session` stack starts its owner at creation and is destroyed when the creating handle's * scope closes or its process exits; a `detached` stack (the default) outlives its creator. @@ -193,7 +193,7 @@ type AnyInstance = ServiceInstances[Kind]; */ export type DestroyResult = | { readonly runtimeCleanup: "complete" } - | { readonly runtimeCleanup: "skipped"; readonly engine: "docker" }; + | { readonly runtimeCleanup: "skipped"; readonly engine: "docker" | "podman" }; /** Options for streaming PostgreSQL command input and output. */ export interface PostgresCommandOptions { readonly args?: ReadonlyArray; @@ -303,7 +303,7 @@ interface Connection { * destroy path) finishes the cleanup once the engine is reachable again. */ const destroyWithoutEngine = Effect.fn("Stack.destroyWithoutEngine")( - function* (state: StackNamespace.Interface, saved: SavedStack, engine: "docker") { + function* (state: StackNamespace.Interface, saved: SavedStack, engine: "docker" | "podman") { const id = saved.id; if (yield* state.leased(id)) return yield* failure( diff --git a/packages/stack/src/host/CommandRunner.ts b/packages/stack/src/host/CommandRunner.ts index a0eabf3faf..d0ba2abc49 100644 --- a/packages/stack/src/host/CommandRunner.ts +++ b/packages/stack/src/host/CommandRunner.ts @@ -76,7 +76,7 @@ const makeCommandRunner = (options: { readonly project?: string; readonly root: string; readonly cacheRoot: string; - readonly runtime: "native" | "docker"; + readonly runtime: "native" | "docker" | "podman"; /** Shares one host-gateway probe with the host's other container runtimes. */ readonly hostGateway?: HostGateway; /** The engine endpoint and identity the owner resolved once at startup; absent when native. */ @@ -329,7 +329,7 @@ export const layer = (options: { readonly project?: string; readonly root: string; readonly cacheRoot: string; - readonly runtime: "native" | "docker"; + readonly runtime: "native" | "docker" | "podman"; /** Shares one host-gateway probe with the host's other container runtimes. */ readonly hostGateway?: HostGateway; /** The engine endpoint and identity the owner resolved once at startup; absent when native. */ diff --git a/packages/stack/src/namespace/Registry.ts b/packages/stack/src/namespace/Registry.ts index c2e1e9f2d5..d3603f1786 100644 --- a/packages/stack/src/namespace/Registry.ts +++ b/packages/stack/src/namespace/Registry.ts @@ -65,7 +65,7 @@ export const SavedStack = Schema.Struct({ branchContext: Schema.String, stackName: Schema.String, }), - runtime: Schema.Literals(["native", "docker"]), + runtime: Schema.Literals(["native", "docker", "podman"]), instances: Schema.Array(SavedInstance).check( Schema.makeFilter((instances) => new Set(instances.map(({ id }) => id)).size === instances.length diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index d85c958bf1..582992dc52 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -118,12 +118,13 @@ const transientPullFailure = (error: ContainerError) => /** Runs one engine CLI invocation outside any pinned target, for resolving that target itself. */ const runRaw = ( spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], + engine: ContainerEngine, args: ReadonlyArray, ): Effect.Effect => Effect.scoped( Effect.gen(function* () { const child = yield* spawner.spawn( - ChildProcess.make("docker", args, { stdin: "ignore", stdout: "pipe", stderr: "pipe" }), + ChildProcess.make(engine, args, { stdin: "ignore", stdout: "pipe", stderr: "pipe" }), ); const [stdout, stderr, code] = yield* Effect.all( [ @@ -144,20 +145,31 @@ const runRaw = ( ), ); +/** The container engine CLI a stack's containers run through. */ +export type ContainerEngine = "docker" | "podman"; + /** * The engine endpoint and identity an owner resolves once, at startup, and pins for its entire * lifetime: the container runtime, the storage helpers, the host-gateway probes and the * namespace's reconcile loop all share this one target instead of each resolving (and so * potentially disagreeing on) their own. `argv` is the explicit prefix every invocation carries - * (`--host `), since the docker CLI gives that priority over `DOCKER_HOST` and context - * switches, and since an argument, unlike an environment variable, never leaks to subprocesses a - * launched workload spawns. + * (`--host ` for Docker, `--url`/`--connection` for Podman), since the CLI gives it + * priority over its environment and saved defaults, and since an argument, unlike an environment + * variable, never leaks to subprocesses a launched workload spawns. `rootless` is whether the + * Podman service runs without root, which decides how host uids map into containers. */ -export interface EngineTarget { - readonly engine: "docker"; - readonly argv: ReadonlyArray; - readonly daemonId: string; -} +export type EngineTarget = + | { + readonly engine: "docker"; + readonly argv: ReadonlyArray; + readonly daemonId: string; + } + | { + readonly engine: "podman"; + readonly argv: ReadonlyArray; + readonly daemonId: string; + readonly rootless: boolean; + }; /** * The active context's own name, pinned by name rather than by its endpoint alone: `context show` @@ -167,26 +179,101 @@ export interface EngineTarget { */ const resolveContextName = (spawner: ChildProcessSpawner.ChildProcessSpawner["Service"]) => Effect.gen(function* () { - const name = (yield* runRaw(spawner, ["context", "show"])).trim(); + const name = (yield* runRaw(spawner, "docker", ["context", "show"])).trim(); if (name.length === 0) return yield* errorFor("context", "Engine returned no active context"); return name; }); +const nonEmptyEnv = (name: string) => + Config.option(Config.string(name)).pipe( + Effect.map(Option.filter((value) => value.length > 0)), + Effect.orElseSucceed(() => Option.none()), + ); + +const PODMAN_INFO_FORMAT = + "{{.Host.ServiceIsRemote}}|{{.Host.Security.Rootless}}|{{.Host.Hostname}}|{{.Store.GraphRoot}}"; + +/** Podman has no daemon `.ID`; its host name and graph root together identify the service. */ +const podmanInfo = ( + spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], + argv: ReadonlyArray, +) => + Effect.gen(function* () { + const output = yield* runRaw(spawner, "podman", [ + ...argv, + "info", + "--format", + PODMAN_INFO_FORMAT, + ]); + const [remote, rootless, hostname, ...graphRoot] = output.trim().split("|"); + const root = graphRoot.join("|"); + if (hostname === undefined || hostname.length === 0 || root.length === 0) + return yield* errorFor("identity", "Engine returned an empty id"); + return { + remote: remote === "true", + rootless: rootless === "true", + daemonId: `${hostname}|${root}`, + }; + }); + +/** + * Pins the connection the Podman CLI would use: `CONTAINER_HOST` or `CONTAINER_CONNECTION` when + * set, otherwise the default saved connection when the CLI talks to a remote service (a + * `podman machine`), and nothing when it talks to a local one, which saved connections never + * redirect. + */ +const resolvePodmanTarget = Effect.fn("Container.resolvePodmanTarget")(function* ( + spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], +): Effect.fn.Return { + const url = yield* nonEmptyEnv("CONTAINER_HOST"); + const connection = yield* nonEmptyEnv("CONTAINER_CONNECTION"); + const explicit: ReadonlyArray | undefined = Option.isSome(url) + ? ["--url", url.value] + : Option.isSome(connection) + ? ["--connection", connection.value] + : undefined; + const bare = explicit === undefined ? yield* podmanInfo(spawner, []) : undefined; + let argv: ReadonlyArray = explicit ?? []; + if (bare?.remote === true) { + const listing = yield* runRaw(spawner, "podman", [ + "system", + "connection", + "list", + "--format", + "{{.Name}}|{{.Default}}", + ]); + const name = listing + .split("\n") + .map((line) => line.trim().split("|")) + .find(([, isDefault]) => isDefault === "true")?.[0]; + if (name === undefined || name.length === 0) + return yield* errorFor("connection", "Podman has no default connection to pin"); + argv = ["--connection", name]; + } + const info = bare !== undefined && argv.length === 0 ? bare : yield* podmanInfo(spawner, argv); + return { engine: "podman", argv, daemonId: info.daemonId, rootless: info.rootless }; +}); + /** - * Resolves and pins, once for an owner's whole lifetime, the single docker endpoint its commands + * Resolves and pins, once for an owner's whole lifetime, the single engine endpoint its commands * target and that endpoint's own identity. */ export const resolveEngineTarget = Effect.fn("Container.resolveEngineTarget")(function* ( spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], + engine: ContainerEngine = "docker", ): Effect.fn.Return { - const host = yield* Config.option(Config.string("DOCKER_HOST")).pipe( - Effect.map(Option.filter((value) => value.length > 0)), - Effect.orElseSucceed(() => Option.none()), - ); + yield* Effect.annotateCurrentSpan("container.engine", engine); + if (engine === "podman") return yield* resolvePodmanTarget(spawner); + const host = yield* nonEmptyEnv("DOCKER_HOST"); const argv: ReadonlyArray = Option.isSome(host) ? ["--host", host.value] : ["--context", yield* resolveContextName(spawner)]; - const daemonId = (yield* runRaw(spawner, [...argv, "info", "--format", "{{.ID}}"])).trim(); + const daemonId = (yield* runRaw(spawner, "docker", [ + ...argv, + "info", + "--format", + "{{.ID}}", + ])).trim(); if (daemonId.length === 0) return yield* errorFor("identity", "Engine returned an empty id"); return { engine: "docker", argv, daemonId }; }); @@ -205,7 +292,12 @@ export const probeEngineReachable = ( spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], target: EngineTarget, ): Effect.Effect => - runRaw(spawner, [...target.argv, "info", "--format", "{{.ID}}"]).pipe( + runRaw(spawner, target.engine, [ + ...target.argv, + "info", + "--format", + target.engine === "docker" ? "{{.ID}}" : "{{.Host.Hostname}}", + ]).pipe( Effect.map((id) => id.trim().length > 0), Effect.orElseSucceed(() => false), ); @@ -226,7 +318,7 @@ const HOST_GATEWAY_PROBE_TIMEOUT: Duration.Input = "15 seconds"; const IPV4_ADDRESS = /^(?:\d{1,3}\.){3}\d{1,3}$/u; /** The name the engine writes into `/etc/hosts` for its own host. */ -const ENGINE_HOST_NAMES = ["host.docker.internal"]; +const ENGINE_HOST_NAMES = ["host.docker.internal", "host.containers.internal"]; const firstIpv4For = (hosts: string, names: ReadonlyArray) => hosts @@ -677,6 +769,11 @@ export const makeContainerRuntime = (options: { `com.docker.compose.service=${composeService}`, ...(oneOff ? ["--label", "com.docker.compose.oneoff=True"] : []), ...(spec.user === undefined ? [] : ["--user", spec.user]), + // A rootless Podman maps the host uid elsewhere inside the container; keep-id maps it to + // itself so files written through a borrowed bind mount stay owned by the caller. + ...(spec.user !== undefined && options.target.engine === "podman" && options.target.rootless + ? ["--userns=keep-id"] + : []), "--env-file", envPath, ...(spec.mounts ?? []).flatMap((mount) => [ @@ -1041,7 +1138,7 @@ const listStackContainers = Effect.fn("Container.listStackContainers")(function* readonly stackRoot: string; }) { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const output = yield* runRaw(spawner, [ + const output = yield* runRaw(spawner, options.target.engine, [ ...options.target.argv, "ps", "--all", diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index 75745f8b27..b815b2cc3b 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -101,7 +101,7 @@ export const DatabaseConfig = Schema.Struct({ const DatabaseReadyMarker = Schema.Struct({ version: Schema.String, - runtime: Schema.Literals(["native", "docker"]), + runtime: Schema.Literals(["native", "docker", "podman"]), profile: Schema.Literal("supabase"), }); @@ -115,7 +115,7 @@ export interface DatabaseEndpoints extends Schema.Schema.Type {} -export type DatabaseRuntime = "native" | "docker"; +export type DatabaseRuntime = "native" | "docker" | "podman"; export type BackendEndpoint = | { readonly kind: "unix"; readonly path: string; readonly port: 5432 } diff --git a/packages/stack/src/services/DatabaseSnapshot.ts b/packages/stack/src/services/DatabaseSnapshot.ts index dfb4d6ffd0..ed6ba06108 100644 --- a/packages/stack/src/services/DatabaseSnapshot.ts +++ b/packages/stack/src/services/DatabaseSnapshot.ts @@ -101,7 +101,7 @@ export interface SnapshotBackend { /** Cache-scoped entries kept besides the one just saved; instance-scoped checkpoints are never pruned. */ const retainedPrevious = 2; const format = "supabase-database-snapshot-v1" as const; -const runtimes = Schema.Literals(["native", "docker"]); +const runtimes = Schema.Literals(["native", "docker", "podman"]); const SnapshotIdentity = Schema.Struct({ format: Schema.Literal(format), version: Schema.String, diff --git a/packages/stack/src/services/Recipe.ts b/packages/stack/src/services/Recipe.ts index b5b975496e..ca652e696b 100644 --- a/packages/stack/src/services/Recipe.ts +++ b/packages/stack/src/services/Recipe.ts @@ -6,7 +6,7 @@ import type { ServiceDefinition } from "../Service.ts"; import type { EngineTarget, HostGateway } from "../runtime/Container.ts"; import type { DockerHelperRegistry } from "../storage/DockerHelperRegistry.ts"; -type CatalogRuntime = "native" | "docker"; +type CatalogRuntime = "native" | "docker" | "podman"; export class CatalogError extends Data.TaggedError("CatalogError")<{ readonly operation: string; diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index 3775c335a1..4f827cc1f1 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -212,6 +212,39 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") cacheNamespace: "native", initialized: false, }; + // Podman has no volume-subpath backend; its data always lives in host directories. + if (options.target.engine === "podman") { + const present = yield* options.fs + .exists(markerPath) + .pipe(Effect.mapError((cause) => errorFor("marker", cause))); + yield* options.fs + .makeDirectory(options.cacheRoot, { recursive: true }) + .pipe(Effect.mapError((cause) => errorFor("identity", cause))); + const cacheRoot = yield* options.fs + .realPath(options.cacheRoot) + .pipe(Effect.mapError((cause) => errorFor("identity", cause))); + const cacheNamespace = `cache-${(yield* hash(cacheRoot)).slice(0, 32)}`; + if (present) { + const marker = yield* options.fs.readFileString(markerPath).pipe( + Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(Marker))), + Effect.mapError((cause) => errorFor("marker", cause)), + Effect.flatMap(validateMarker), + ); + if (marker.cacheNamespace === cacheNamespace) return marker; + const updated = { ...marker, cacheNamespace }; + yield* publishMarker(yield* encodeMarker(updated)); + return updated; + } + yield* rejectUnmarkedData; + const value: Marker = { + backend: "host", + namespace: dataNamespace, + cacheNamespace, + initialized: false, + }; + yield* publishMarker(yield* encodeMarker(value)); + return value; + } let resolved = yield* resolveDaemon(false); const markerPresent = yield* options.fs .exists(markerPath) @@ -332,12 +365,12 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") Effect.scoped( Effect.gen(function* () { yield* Effect.annotateCurrentSpan({ - "process.executable.name": options.runtime, + "process.executable.name": options.target.engine, "process.arg_count": args.length, }); const child = yield* options.spawner .spawn( - ChildProcess.make(options.runtime, [...options.target.argv, ...args], { + ChildProcess.make(options.target.engine, [...options.target.argv, ...args], { stdin: "ignore", }), ) @@ -600,7 +633,10 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") "trap : TERM INT; while :; do sleep 3600; done", ]); if (!/^[a-f0-9]{12,64}$/u.test(created)) - return yield* errorFor("helper", "Docker returned an invalid helper identity"); + return yield* errorFor( + "helper", + `${options.target.engine === "docker" ? "Docker" : "Podman"} returned an invalid helper identity`, + ); return name; }), ), @@ -771,7 +807,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") const cached = yield* resolvedFromMemory; if (cached !== undefined) return Option.some(cached); const existing = yield* getMarkerIfPresent; - if (Option.isSome(existing) && options.runtime === "docker") { + if (Option.isSome(existing) && options.runtime !== "native") { yield* selected; return Option.some(yield* getMarker); } diff --git a/packages/stack/src/testing.ts b/packages/stack/src/testing.ts index 2be287bc91..b92c16d860 100644 --- a/packages/stack/src/testing.ts +++ b/packages/stack/src/testing.ts @@ -114,7 +114,7 @@ export interface TestStack< } const runtimeOverride = Config.option( - Config.literals(["native", "docker"], "SUPABASE_STACK_TEST_RUNTIME"), + Config.literals(["native", "docker", "podman"], "SUPABASE_STACK_TEST_RUNTIME"), ); const testFailure = (operation: string) => (cause: unknown) => stackError(operation, cause); From c6bcab064e0ea9feae2efff76f21d221a2681365 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 13:14:49 +0200 Subject: [PATCH 089/142] feat(cli): select Podman for stacks when Docker is unavailable --- apps/cli/docs/stack-commands.md | 18 ++--- apps/cli/src/command-internal/stack-api.ts | 12 +++- .../command-internal/stack-local-database.ts | 3 +- .../cli/src/command-internal/stack-runtime.ts | 67 ++++++++----------- .../stack-runtime.unit.test.ts | 6 +- apps/cli/src/command-internal/stack-shadow.ts | 2 +- ...elta-next-shadow.stack.integration.test.ts | 4 +- .../stack/destroy/SIDE_EFFECTS.md | 6 +- .../stack/destroy/destroy.handler.ts | 7 +- ...oy.runtime-unavailable.integration.test.ts | 20 +++++- .../experimental/stack/list/SIDE_EFFECTS.md | 4 +- .../stack/prepare/SIDE_EFFECTS.md | 2 +- .../stack/prepare/prepare.command.ts | 4 +- .../stack/prepare/prepare.integration.test.ts | 56 +++++++++++----- .../experimental/stack/start/SIDE_EFFECTS.md | 17 ++--- .../experimental/stack/start/start.command.ts | 6 +- .../experimental/stack/start/start.handler.ts | 26 ++++--- .../stack/start/start.integration.test.ts | 56 ++++++++++++++++ .../stack/status/status.handler.ts | 5 +- .../tests/helpers/child-process-spawner.ts | 27 ++++++-- packages/stack/package.json | 1 + packages/stack/src/internal/engine.ts | 2 + 22 files changed, 237 insertions(+), 114 deletions(-) create mode 100644 packages/stack/src/internal/engine.ts diff --git a/apps/cli/docs/stack-commands.md b/apps/cli/docs/stack-commands.md index abb32853aa..0e7e194a62 100644 --- a/apps/cli/docs/stack-commands.md +++ b/apps/cli/docs/stack-commands.md @@ -2,8 +2,8 @@ `supabase stack` manages local stacks with the new experimental runtime. It is unstable, its command interface may change, and it is excluded from the CLI compatibility promise. It is -available when the `experimental.stack` feature flag is enabled and supports Docker and native -runtimes. +available when the `experimental.stack` feature flag is enabled and supports Docker, Podman, and +native runtimes. Native PostgreSQL requires passwords for every role except `supabase_admin`. Its local bootstrap and password-reconciliation connection uses that administrative role, so a native @@ -22,13 +22,13 @@ and password-reconciliation connection uses that administrative role, so a nativ Use each command's `--help` for its available targeting and runtime options. -A new stack created with `--runtime auto` uses Docker when its daemon answers, and otherwise native -on Linux x64/arm64 and macOS arm64. On other platforms without a reachable engine, the command -fails and asks you to start Docker. The selected runtime is saved with the stack and reused without -probing; when auto selection skips Docker, the command prints a notice saying so. To switch, -destroy the stack or choose a different -`--stack` name. When an explicit `--runtime docker` or a saved Docker stack cannot reach Docker, -the failure asks you to install or start it, and also suggests `--runtime native` for a new stack +A new stack created with `--runtime auto` uses Docker when its daemon answers, otherwise Podman +when its engine answers, and otherwise native on Linux x64/arm64 and macOS arm64. On other +platforms without a reachable engine, the command fails and asks you to start Docker or Podman. The +selected runtime is saved with the stack and reused without probing; when auto selection skips +Docker, the command prints a notice saying so. To switch, destroy the stack or choose a different +`--stack` name. When an explicit `--runtime docker` or `--runtime podman`, or a saved container +stack, cannot reach its engine, the failure asks you to install or start it, and also suggests `--runtime native` for a new stack on platforms that support native. Project stacks created by database commands, and shadow stacks created without a project stack, use the same selection. diff --git a/apps/cli/src/command-internal/stack-api.ts b/apps/cli/src/command-internal/stack-api.ts index 03d2497a11..88780e5c45 100644 --- a/apps/cli/src/command-internal/stack-api.ts +++ b/apps/cli/src/command-internal/stack-api.ts @@ -1,3 +1,4 @@ +import type { ContainerEngine } from "@supabase/stack/internal/engine"; import { Context, Crypto, Effect, FileSystem, Layer, Option, Path, Scope } from "effect"; import { FetchHttpClient, HttpClient } from "effect/unstable/http"; import { ChildProcessSpawner } from "effect/unstable/process"; @@ -13,6 +14,7 @@ import { type Stack, type StackError, } from "@supabase/stack/effect"; +import { containerEngineName } from "./stack-runtime.ts"; type DiscoverResult = Effect.Success>; @@ -67,7 +69,13 @@ export const stackApiLayer = Layer.effect( ).pipe(Layer.provide(FetchHttpClient.layer)); /** Describes why a destroy left engine resources behind, without promising automatic cleanup. */ -export const skippedRuntimeCleanupWarning = (subject: string, stackId: string): string => - `Docker was unavailable, so Docker resources for ${subject} were not removed. Restore Docker and run "supabase stack destroy --stack-id ${stackId}" again to finish removing it.`; +export const skippedRuntimeCleanupWarning = ( + subject: string, + stackId: string, + engine: ContainerEngine, +): string => { + const name = containerEngineName(engine); + return `${name} was unavailable, so ${name} resources for ${subject} were not removed. Restore ${name} and run "supabase stack destroy --stack-id ${stackId}" again to finish removing it.`; +}; export type { Stack }; diff --git a/apps/cli/src/command-internal/stack-local-database.ts b/apps/cli/src/command-internal/stack-local-database.ts index 8ecda0e7b8..9af7a359dd 100644 --- a/apps/cli/src/command-internal/stack-local-database.ts +++ b/apps/cli/src/command-internal/stack-local-database.ts @@ -24,6 +24,7 @@ import { automaticRuntimeNotice, recordStackRuntimeTelemetry, selectStackRuntime, + type StackRuntime, } from "./stack-runtime.ts"; import { Output } from "../shared/output/output.service.ts"; import type { PgConnInput } from "./db-connection.service.ts"; @@ -127,7 +128,7 @@ const RUNTIME_UNAVAILABLE = new StackRuntimeUnavailableError({ }); export const stackRequireProjectRuntime: Effect.Effect< - "native" | "docker", + StackRuntime, StackRuntimeUnavailableError, CommandSettings | StackApi | Path.Path > = stackProjectRuntime.pipe( diff --git a/apps/cli/src/command-internal/stack-runtime.ts b/apps/cli/src/command-internal/stack-runtime.ts index 1e4ee9c1db..4d1144e1fc 100644 --- a/apps/cli/src/command-internal/stack-runtime.ts +++ b/apps/cli/src/command-internal/stack-runtime.ts @@ -1,6 +1,6 @@ import { defaultRuntime } from "@supabase/stack/internal/artifacts"; +import { resolveEngineTarget, type ContainerEngine } from "@supabase/stack/internal/engine"; import { Data, Effect } from "effect"; -import * as ChildProcess from "effect/unstable/process/ChildProcess"; import { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner"; import { RuntimeInfo } from "../shared/runtime/runtime-info.service.ts"; import { @@ -9,11 +9,10 @@ import { ErrorActionabilityId, } from "../shared/telemetry/error-actionability.ts"; import { PropStackRuntime } from "../shared/telemetry/event-catalog.ts"; -import { withChildTraceEnv, withProcessSpan } from "../shared/telemetry/spans.ts"; import { recordCommandTelemetry } from "../telemetry/command-telemetry-attributes.ts"; /** Runtime that executes a local stack's services. */ -export type StackRuntime = "native" | "docker"; +export type StackRuntime = "native" | "docker" | "podman"; /** Records the runtime of the stack the enclosing command operates on. */ export const recordStackRuntimeTelemetry = (runtime: StackRuntime) => @@ -32,42 +31,30 @@ export class StackRuntimeSelectionError extends Data.TaggedError("StackRuntimeSe } } -/** A cold Docker Desktop machine can take several seconds to answer its first request. */ +/** A cold Docker Desktop or Podman machine can take several seconds to answer its first request. */ const PROBE_TIMEOUT = "10 seconds"; -/** Each probe exits non-zero unless the engine's daemon answers. */ -const engineProbes = [ - { runtime: "docker", args: ["version", "--format", "{{.Server.Version}}"] }, -] as const; - -const engineReachable = ( +/** + * Resolves the engine target the owner would pin, so selection and startup agree on which endpoint + * answers; resolution reads the engine's identity, so success means the engine is reachable. + */ +const engineReachable = Effect.fn("StackRuntime.probeEngine")(function* ( spawner: ChildProcessSpawner["Service"], - probe: (typeof engineProbes)[number], -) => - withProcessSpan( - "StackRuntime.probeEngine", - { - executable: probe.runtime, - argCount: probe.args.length, - subcommand: probe.args[0], - hasFallback: true, - }, - (traceEnv) => - spawner.exitCode( - ChildProcess.make( - probe.runtime, - probe.args, - withChildTraceEnv( - { stdin: "ignore", stdout: "ignore", stderr: "ignore", forceKillAfter: "1 second" }, - traceEnv, - ), - ), - ), - ).pipe( + engine: ContainerEngine, +) { + yield* Effect.annotateCurrentSpan({ "container.engine": engine }); + const reachable = yield* resolveEngineTarget(spawner, engine).pipe( + Effect.as(true), Effect.timeout(PROBE_TIMEOUT), - Effect.map((exitCode) => exitCode === 0), Effect.orElseSucceed(() => false), ); + yield* Effect.annotateCurrentSpan({ "container.reachable": reachable }); + return reachable; +}); + +/** Display name of a container runtime. */ +export const containerEngineName = (runtime: ContainerEngine) => + runtime === "docker" ? "Docker" : "Podman"; /** * Notice for a new stack whose automatic selection skipped Docker, since that runtime is saved with @@ -79,9 +66,9 @@ export const automaticRuntimeNotice = ( ): string | undefined => requested !== undefined || selected === "docker" ? undefined - : `Docker didn't answer, so this new stack uses the native runtime, which is saved with the stack. To use Docker, start it, then run \`supabase stack destroy\` and start again, or choose a different --stack name with --runtime docker.`; + : `Docker didn't answer, so this new stack uses the ${selected === "podman" ? "Podman" : "native"} runtime, which is saved with the stack. To use Docker, start it, then run \`supabase stack destroy\` and start again, or choose a different --stack name with --runtime docker.`; -/** Returns the requested or saved runtime unchanged; otherwise Docker, or native, if usable. */ +/** Returns the requested or saved runtime unchanged; otherwise the first of Docker, Podman, or native that is usable on this host. */ export const selectStackRuntime = Effect.fn("StackRuntime.select")(function* ( requested: StackRuntime | undefined, ) { @@ -94,20 +81,20 @@ export const selectStackRuntime = Effect.fn("StackRuntime.select")(function* ( reason: "native-unsupported", message: `Native artifacts are unsupported on ${platform}/${arch}.`, suggestion: - "Start Docker and rerun with --runtime docker; if this stack already exists as native, run supabase stack destroy first.", + "Start Docker or Podman and rerun with --runtime docker or --runtime podman; if this stack already exists as native, run supabase stack destroy first.", }); } return requested; } const spawner = yield* ChildProcessSpawner; - for (const probe of engineProbes) { - if (yield* engineReachable(spawner, probe)) return probe.runtime; + for (const engine of ["docker", "podman"] as const) { + if (yield* engineReachable(spawner, engine)) return engine; } const { platform, arch } = yield* RuntimeInfo; if (defaultRuntime({ os: platform, arch }) === "native") return "native"; return yield* new StackRuntimeSelectionError({ reason: "engine-unreachable", - message: `Docker is not reachable, and native stacks are not supported on ${platform}/${arch}.`, - suggestion: "Start Docker, then rerun the command.", + message: `Neither Docker nor Podman is reachable, and native stacks are not supported on ${platform}/${arch}.`, + suggestion: "Start Docker or Podman, then rerun the command.", }); }, Effect.tap(recordStackRuntimeTelemetry)); diff --git a/apps/cli/src/command-internal/stack-runtime.unit.test.ts b/apps/cli/src/command-internal/stack-runtime.unit.test.ts index 7b44291715..79794d8306 100644 --- a/apps/cli/src/command-internal/stack-runtime.unit.test.ts +++ b/apps/cli/src/command-internal/stack-runtime.unit.test.ts @@ -13,14 +13,14 @@ describe("selectStackRuntime", () => { Effect.gen(function* () { const recorded = yield* Ref.make({}); - const runtime = yield* selectStackRuntime("docker").pipe( + const runtime = yield* selectStackRuntime("podman").pipe( Effect.provideService(CommandTelemetryAttributes, { record: (values) => Ref.update(recorded, (current) => ({ ...current, ...values })), }), ); - expect(runtime).toBe("docker"); - expect((yield* Ref.get(recorded)).stack_runtime).toBe("docker"); + expect(runtime).toBe("podman"); + expect((yield* Ref.get(recorded)).stack_runtime).toBe("podman"); }).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, runtimeInfoLayer))), ); }); diff --git a/apps/cli/src/command-internal/stack-shadow.ts b/apps/cli/src/command-internal/stack-shadow.ts index 363fa9ccc2..729d5a77cf 100644 --- a/apps/cli/src/command-internal/stack-shadow.ts +++ b/apps/cli/src/command-internal/stack-shadow.ts @@ -236,7 +236,7 @@ export const stackAcquireShadowDatabase = Effect.fn("StackShadow.acquire")(funct Effect.flatMap((result) => result.runtimeCleanup === "skipped" ? output.raw( - `Warning: ${skippedRuntimeCleanupWarning(`shadow stack ${stack.id}`, stack.id)}\n`, + `Warning: ${skippedRuntimeCleanupWarning(`shadow stack ${stack.id}`, stack.id, result.engine)}\n`, "stderr", ) : Effect.void, diff --git a/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts b/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts index 2ee1e255ab..9b4a0acf11 100644 --- a/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts +++ b/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts @@ -92,7 +92,7 @@ describe("pg-delta next stack shadow provisioning", () => { ); const stateRoot = `${root}/stacks`; - const engines = containerEngineSpawner({ docker: "missing" }); + const engines = containerEngineSpawner({ docker: "missing", podman: "missing" }); const settings = mockCommandSettings({ workdir: root, supabaseHome: root }); const output = mockOutput().layer; const apiLayer = stackApiLayer.pipe( @@ -155,7 +155,7 @@ describe("pg-delta next stack shadow provisioning", () => { ).toEqual([{ table_name: null }]); const api = yield* StackApi; expect(yield* api.discover({ stateRoot })).toHaveLength(2); - expect(engines.spawned.map(({ command }) => command)).toEqual(["docker"]); + expect(engines.spawned.map(({ command }) => command)).toEqual(["docker", "podman"]); return plan; }).pipe(Effect.provide(services)), ); diff --git a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md index 71d98af30e..3aa9e93e06 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md @@ -29,9 +29,9 @@ shutdown is refused. When no owner is running and the engine reports that its daemon cannot be reached, destruction removes nothing: the registration, port claims and host -data stay as they are, stderr warns that Docker resources were not removed, and +data stay as they are, stderr warns that the stack's engine (Docker or Podman) resources were not removed, and the command exits 0 reporting the stack as not destroyed. Run destroy again once -Docker is reachable. Any other engine failure, an owner starting during +the engine is reachable. Any other engine failure, an owner starting during destruction, an instance whose data (including a native database's socket directory) cannot be removed, or a labelled container that survives removal, fails the command and keeps the stack registered; the error lists what @@ -62,7 +62,7 @@ remains the backstop when no owner is left alive to do so. ## Output, exit codes and telemetry Text prints `Stack destroyed.`, or, when engine cleanup was skipped, -`Stack could not be fully destroyed because Docker is unreachable; restore it and run "supabase stack destroy --stack-id " again.` +`Stack could not be fully destroyed because is unreachable; restore it and run "supabase stack destroy --stack-id " again.` JSON and stream-json success data contain `destroyed` (`false` when engine cleanup was skipped), `id`, and `runtimeCleanup` (`complete` or `skipped`); a skipped cleanup also carries `engine`. Exit 0 on destruction or skipped engine cleanup, 1 on invalid flags, missing diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts index be36d020eb..2d3ffb13c2 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts @@ -14,6 +14,7 @@ import { mapTargetError, validateStackTarget, } from "../stack.shared.ts"; +import { containerEngineName } from "../../../../command-internal/stack-runtime.ts"; import type { StackDestroyFlags } from "./destroy.command.ts"; import { StackCommandDestroyError } from "./destroy.errors.ts"; @@ -106,7 +107,9 @@ export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( "stack.runtime_cleanup": result.runtimeCleanup, }); if (result.runtimeCleanup === "skipped") - yield* output.warn(skippedRuntimeCleanupWarning(`stack ${target.id}`, target.id)); + yield* output.warn( + skippedRuntimeCleanupWarning(`stack ${target.id}`, target.id, result.engine), + ); if (output.format !== "text") yield* output.success("", { destroyed: result.runtimeCleanup === "complete", @@ -117,7 +120,7 @@ export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( yield* output.raw(`Stack ${target.id} destroyed.\n`); else yield* output.raw( - `Stack ${target.id} could not be fully destroyed because Docker is unreachable; restore it and run "supabase stack destroy --stack-id ${target.id}" again.\n`, + `Stack ${target.id} could not be fully destroyed because ${containerEngineName(result.engine)} is unreachable; restore it and run "supabase stack destroy --stack-id ${target.id}" again.\n`, ); }); return yield* body.pipe(Effect.ensuring(telemetryState.flush)); diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts index 0069203600..7185076daa 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts @@ -21,20 +21,21 @@ const live = Layer.provideMerge(stackApiLayer, BunServices.layer); */ const fixture = Effect.fn("StackDestroyRuntimeUnavailableTest.fixture")(function* ( format: "text" | "json", + engine: "docker" | "podman" = "docker", ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-runtime-" }); const realApi = yield* StackApi; const locations = { stateRoot: path.join(root, "stacks"), cacheRoot: path.join(root, "cache") }; - const stack = yield* realApi.create({ ...locations, projectRoot: root, runtime: "docker" }); + const stack = yield* realApi.create({ ...locations, projectRoot: root, runtime: engine }); const api = StackApi.of({ ...realApi, open: (options) => realApi.open(options).pipe( Effect.map((opened) => ({ ...opened, - destroy: Effect.succeed({ runtimeCleanup: "skipped", engine: "docker" } as const), + destroy: Effect.succeed({ runtimeCleanup: "skipped", engine } as const), })), ), }); @@ -98,3 +99,18 @@ it.live("reports skipped engine cleanup in the JSON result", () => expect(yield* f.api.discover(f.locations)).toHaveLength(1); }).pipe(Effect.scoped, Effect.provide(live)), ); + +it.live("names Podman when a Podman stack's engine is unreachable", () => + Effect.gen(function* () { + const f = yield* fixture("text", "podman"); + yield* stackDestroy(f.flags).pipe(Effect.provide(f.layer)); + expect(f.output.stdoutText).toBe( + `Stack ${f.stack.id} could not be fully destroyed because Podman is unreachable; restore it and run "supabase stack destroy --stack-id ${f.stack.id}" again.\n`, + ); + expect(f.output.messages).toContainEqual({ + type: "warn", + message: `Podman was unavailable, so Podman resources for stack ${f.stack.id} were not removed. Restore Podman and run "supabase stack destroy --stack-id ${f.stack.id}" again to finish removing it.`, + }); + expect(yield* f.api.discover(f.locations)).toHaveLength(1); + }).pipe(Effect.scoped, Effect.provide(live)), +); diff --git a/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md index 6d382077df..235da57650 100644 --- a/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md @@ -17,8 +17,8 @@ port or endpoint information. Telemetry flushes to ## Output -Entries contain `id`, `project_root`, `name`, `branch_context`, `runtime` (native -or docker), and `owner` (reachable or unavailable). Owner availability is +Entries contain `id`, `project_root`, `name`, `branch_context`, `runtime` (native, +docker, or podman), and `owner` (reachable or unavailable). Owner availability is not service lifecycle or health. Entries sort by project root, name, then ID. Text shows NAME, PROJECT, BRANCH, RUNTIME, OWNER, and a compact ID. An empty registry prints `No managed stacks found.` Use `--output-format json` to obtain diff --git a/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md index 7d26c5f7ac..4d51539261 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md @@ -35,7 +35,7 @@ caching, preparation, and cancellation; the CLI owns the temporary instance clea ## Flags and output `--stack` and `--stack-id` are mutually exclusive. `--runtime` defaults to `auto` for new stacks, -which selects the same runtime as `stack start` (Docker, then native); +which selects the same runtime as `stack start` (Docker, then Podman, then native); existing stacks reuse their persisted runtime without probing, and an explicit mismatch fails. With no `--capability`, every enabled creation from the effective project configuration is prepared. Repeated `--capability` includes each capability's configured companion services (Storage/Imgproxy, diff --git a/apps/cli/src/commands/experimental/stack/prepare/prepare.command.ts b/apps/cli/src/commands/experimental/stack/prepare/prepare.command.ts index 0d91f3e85b..6ea91d399d 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/prepare.command.ts +++ b/apps/cli/src/commands/experimental/stack/prepare/prepare.command.ts @@ -11,9 +11,9 @@ const config = { Flag.withDescription("Open an existing stack by id."), Flag.optional, ), - runtime: Flag.choice("runtime", ["auto", "docker", "native"] as const).pipe( + runtime: Flag.choice("runtime", ["auto", "docker", "podman", "native"] as const).pipe( Flag.withDescription( - "Runtime to use for a new stack. auto selects Docker, then native, based on what is available.", + "Runtime to use for a new stack. auto selects Docker, then Podman, then native, based on what is available.", ), Flag.withDefault("auto" as const), ), diff --git a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts index 847a5259e1..e89295c1dc 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts @@ -49,13 +49,18 @@ const makeProject = (config: string) => interface FixtureOptions { readonly failPreparation?: boolean; readonly savedRuntime?: StackRuntime; - readonly engines?: { readonly docker: ContainerEngineState }; + readonly engines?: { + readonly docker: ContainerEngineState; + readonly podman: ContainerEngineState; + }; readonly runtimeInfo?: Layer.Layer; } const makeFixture = (root: string, options: FixtureOptions = {}) => { const { failPreparation = false } = options; - const engines = containerEngineSpawner(options.engines ?? { docker: "missing" }); + const engines = containerEngineSpawner( + options.engines ?? { docker: "missing", podman: "missing" }, + ); const createdRuntimes: Array = []; let openCount = 0; let prepareCount = 0; @@ -205,7 +210,7 @@ const makeFixture = (root: string, options: FixtureOptions = {}) => { .map(({ message }) => message); }, get probes() { - return engines.spawned.map(({ command }) => command); + return [...new Set(engines.spawned.map(({ command }) => command))]; }, get prepareCount() { return prepareCount; @@ -343,7 +348,7 @@ describe("stack prepare automatic runtime selection", () => { ); it.live("creates a Docker stack when the Docker engine answers", () => - selectRuntime({ engines: { docker: "running" } }).pipe( + selectRuntime({ engines: { docker: "running", podman: "running" } }).pipe( Effect.tap((fixture) => Effect.sync(() => { expect(fixture.createdRuntimes).toEqual(["docker"]); @@ -354,22 +359,41 @@ describe("stack prepare automatic runtime selection", () => { ), ); - it.live("creates a native stack on Linux when the Docker daemon is down", () => + it.live("creates a Podman stack when Docker is not installed and Podman answers", () => selectRuntime({ - engines: { docker: "stopped" }, + engines: { docker: "missing", podman: "running" }, runtimeInfo: mockRuntimeInfo({ platform: "linux", arch: "x64" }), }).pipe( Effect.tap((fixture) => Effect.sync(() => { - expect(fixture.createdRuntimes).toEqual(["native"]); - expect(fixture.probes).toEqual(["docker"]); + expect(fixture.createdRuntimes).toEqual(["podman"]); + expect(fixture.probes).toEqual(["docker", "podman"]); expect(fixture.runtimeNotices).toHaveLength(1); - expect(fixture.runtimeNotices[0]).toContain("uses the native runtime"); + expect(fixture.runtimeNotices[0]).toContain("uses the Podman runtime"); + expect(fixture.runtimeNotices[0]).toContain("supabase stack destroy"); }), ), ), ); + it.live( + "creates a native stack on Linux when the Docker daemon is down and Podman is absent", + () => + selectRuntime({ + engines: { docker: "stopped", podman: "missing" }, + runtimeInfo: mockRuntimeInfo({ platform: "linux", arch: "x64" }), + }).pipe( + Effect.tap((fixture) => + Effect.sync(() => { + expect(fixture.createdRuntimes).toEqual(["native"]); + expect(fixture.probes).toEqual(["docker", "podman"]); + expect(fixture.runtimeNotices).toHaveLength(1); + expect(fixture.runtimeNotices[0]).toContain("uses the native runtime"); + }), + ), + ), + ); + for (const [platform, arch] of [ ["win32", "x64"], ["darwin", "x64"], @@ -378,7 +402,7 @@ describe("stack prepare automatic runtime selection", () => { makeProject(databaseOnlyConfig).pipe( Effect.flatMap((root) => { const fixture = makeFixture(root, { - engines: { docker: "stopped" }, + engines: { docker: "stopped", podman: "stopped" }, runtimeInfo: mockRuntimeInfo({ platform, arch }), }); return stackPrepare(flags({ runtime: "auto" })).pipe( @@ -390,7 +414,7 @@ describe("stack prepare automatic runtime selection", () => { expect(error.reason).toBe("runtime"); expect(error.cause).toBeInstanceOf(StackRuntimeSelectionError); expect(error.message).toContain(`not supported on ${platform}/${arch}`); - expect(error.suggestion).toBe("Start Docker, then rerun the command."); + expect(error.suggestion).toBe("Start Docker or Podman, then rerun the command."); expect(fixture.createdRuntimes).toEqual([]); }), ), @@ -403,8 +427,8 @@ describe("stack prepare automatic runtime selection", () => { it.live("reuses a saved runtime without probing any engine", () => selectRuntime({ - savedRuntime: "docker", - engines: { docker: "running" }, + savedRuntime: "podman", + engines: { docker: "running", podman: "running" }, }).pipe( Effect.tap((fixture) => Effect.sync(() => { @@ -420,12 +444,12 @@ describe("stack prepare automatic runtime selection", () => { it.live("honors an explicit runtime without probing any engine", () => makeProject(databaseOnlyConfig).pipe( Effect.flatMap((root) => { - const fixture = makeFixture(root, { engines: { docker: "stopped" } }); - return stackPrepare(flags({ runtime: "docker" })).pipe( + const fixture = makeFixture(root, { engines: { docker: "running", podman: "running" } }); + return stackPrepare(flags({ runtime: "podman" })).pipe( Effect.provide(fixture.layer), Effect.tap(() => Effect.sync(() => { - expect(fixture.createdRuntimes).toEqual(["docker"]); + expect(fixture.createdRuntimes).toEqual(["podman"]); expect(fixture.probes).toEqual([]); expect(fixture.runtimeNotices).toEqual([]); }), diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index c7fdc361d4..8667ec77a1 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -50,14 +50,15 @@ auto range and below the OS ephemeral range on every supported platform; this ra configurable, and a public port pinned inside it is rejected. A host whose ephemeral range has been widened to overlap it reintroduces the ephemeral-port race this reservation exists to avoid. -For a new stack, `--runtime auto` selects Docker when `docker version` reaches its daemon, then -native on Linux x64/arm64 and macOS arm64. The probe is bounded by 10 seconds. Without a reachable -engine on other platforms, the command fails and asks the user to start Docker. When auto selection -skips Docker, an info line names the saved native runtime and how to switch to Docker. An existing -stack keeps its saved runtime and runs no probe. Explicit `--runtime docker` or `native` has no -fallback. -When an explicit or saved Docker runtime is unreachable, the reported failure suggests starting -Docker, and `--runtime native` for a new stack on platforms that support native. Explicit +For a new stack, `--runtime auto` selects Docker when its engine answers, then Podman when its +engine answers, then native on Linux x64/arm64 and macOS arm64. Each probe resolves the engine +target the owner later pins (so a local or remote Podman is judged by the same endpoint) and is +bounded by 10 seconds. Without a reachable engine on other platforms, the command fails and asks +the user to start Docker or Podman. When auto selection skips Docker, an info line names the saved +Podman or native runtime and how to switch to Docker. An existing stack keeps its saved runtime and +runs no probe. Explicit `--runtime docker`, `podman`, or `native` has no fallback. +When an explicit or saved container runtime is unreachable, the reported failure suggests starting +that engine, and `--runtime native` for a new stack on platforms that support native. Explicit `--runtime native` on a platform with no native artifacts fails before creating a stack. Native startup refuses root because PostgreSQL `initdb` cannot run as root, unless a Claude Code diff --git a/apps/cli/src/commands/experimental/stack/start/start.command.ts b/apps/cli/src/commands/experimental/stack/start/start.command.ts index 8b569790ca..e5e7363d16 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.command.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.command.ts @@ -22,9 +22,9 @@ const config = { Flag.withDescription("Open an existing stack by id."), Flag.optional, ), - runtime: Flag.choice("runtime", ["auto", "docker", "native"] as const).pipe( + runtime: Flag.choice("runtime", ["auto", "docker", "podman", "native"] as const).pipe( Flag.withDescription( - "Runtime to use for a new stack. auto selects Docker, then native, based on what is available.", + "Runtime to use for a new stack. auto selects Docker, then Podman, then native, based on what is available.", ), Flag.withDefault("auto" as const), ), @@ -46,7 +46,7 @@ export const stackStartCommand = Command.make("start", config).pipe( Command.withDescription( "Create or resume a managed local Supabase stack using supabase/config.toml when present. " + "Without a config file, default settings are used and no file is created. " + - "For a new stack, auto selects Docker when its engine is reachable, then native on supported platforms; the choice is persisted for that stack. " + + "For a new stack, auto selects Docker when its engine is reachable, then Podman, then native on supported platforms; the choice is persisted for that stack. " + "Explicit runtime choices are honored. Values support explicit env(NAME) references and automatic SUPABASE_* overrides.", ), Command.withShortDescription("Start a managed local stack"), diff --git a/apps/cli/src/commands/experimental/stack/start/start.handler.ts b/apps/cli/src/commands/experimental/stack/start/start.handler.ts index 1c36f221f2..9243573afa 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.handler.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.handler.ts @@ -13,6 +13,7 @@ import { withProjectFunctionsEnv } from "../../../../command-internal/stack-func import { statusEnvPointer } from "./start-summary.format.ts"; import { automaticRuntimeNotice, + containerEngineName, selectStackRuntime, type StackRuntime, } from "../../../../command-internal/stack-runtime.ts"; @@ -104,13 +105,18 @@ const stackError = ( }; // A saved stack keeps its runtime, so only a new stack can switch to native. -const dockerUnavailableSuggestion = ( +const engineUnavailableSuggestion = ( + engine: Exclude, runtimeInfo: { readonly platform: string; readonly arch: string }, creating: boolean, -) => - creating && defaultRuntime({ os: runtimeInfo.platform, arch: runtimeInfo.arch }) === "native" - ? "Docker CLI or daemon isn't reachable. Install or start Docker, or run with --runtime native." - : "Docker CLI or daemon isn't reachable. Install or start Docker."; +) => { + const name = containerEngineName(engine); + const base = `${name} CLI or daemon isn't reachable. Install or start ${name}`; + return creating && + defaultRuntime({ os: runtimeInfo.platform, arch: runtimeInfo.arch }) === "native" + ? `${base}, or run with --runtime native.` + : `${base}.`; +}; const stackAcquireError = ( cause: StackError, @@ -121,13 +127,17 @@ const stackAcquireError = ( }, ) => { const base = stackError(cause); - if (cause.reason !== "runtime-unavailable" || runtimeContext.selectedRuntime !== "docker") - return base; + const { selectedRuntime } = runtimeContext; + if (cause.reason !== "runtime-unavailable" || selectedRuntime === "native") return base; return new StackCommandStartError({ reason: "runtime", message: base.message, ...(base.detail === undefined ? {} : { detail: base.detail }), - suggestion: dockerUnavailableSuggestion(runtimeContext.runtime, runtimeContext.creating), + suggestion: engineUnavailableSuggestion( + selectedRuntime, + runtimeContext.runtime, + runtimeContext.creating, + ), cause: base.cause, }); }; diff --git a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts index e1dbe2351b..9da7a7a7c6 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts @@ -35,6 +35,7 @@ import { mockTelemetryStateTracked, withEnvVar, } from "../../../../../tests/helpers/command-mocks.ts"; +import { containerEngineSpawner } from "../../../../../tests/helpers/child-process-spawner.ts"; import { mockOutput, mockProcessControl, @@ -619,6 +620,61 @@ describe("experimental stack start", () => { }).pipe(Effect.provide(BunServices.layer)), ); + it.live( + "creates and reports a Podman stack when automatic selection skips a stopped Docker", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-auto-podman-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "auto-podman"\n[edge_runtime]\nenabled = false\n', + ); + const output = mockOutput(); + const engines = containerEngineSpawner({ docker: "stopped", podman: "running" }); + const fixture = fakeStack(); + const createdRuntimes: Array = []; + const overrides = Layer.mergeAll( + Layer.succeed(StackTargetResolver, { + resolve: () => Effect.succeed({ projectRoot: root, hostRunning: false }), + }), + Layer.succeed(StackApi, { + create: (options) => + Effect.sync(() => { + createdRuntimes.push(options.runtime); + return fixture.stack; + }), + open: () => Effect.succeed(fixture.stack), + discover: () => Effect.succeed([]), + find: () => Effect.die("identity not used"), + }), + engines.layer, + ); + yield* stackStart({ + ...flags([ + "rest", + "auth", + "realtime", + "storage", + "functions", + "studio", + "mail", + "analytics", + "pooler", + ]), + runtime: "auto", + }).pipe(Effect.provide(Layer.merge(layers(root, fixture, output, false), overrides))); + expect(createdRuntimes).toEqual(["podman"]); + expect(output.messages).toContainEqual({ + type: "info", + message: expect.stringContaining( + "Docker didn't answer, so this new stack uses the Podman runtime", + ), + }); + }).pipe(Effect.provide(BunServices.layer)), + ); + it.live("applies capability selection across repeated starts", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/cli/src/commands/experimental/stack/status/status.handler.ts b/apps/cli/src/commands/experimental/stack/status/status.handler.ts index ef57a9683a..cdfaa5c9f1 100644 --- a/apps/cli/src/commands/experimental/stack/status/status.handler.ts +++ b/apps/cli/src/commands/experimental/stack/status/status.handler.ts @@ -6,6 +6,7 @@ import { Output } from "../../../../shared/output/output.service.ts"; import { OutputFlag } from "../../../../command-internal/global-flags.ts"; import { CommandSettings } from "../../../../config/command-settings.service.ts"; import { TelemetryState } from "../../../../telemetry/telemetry-state.service.ts"; +import type { StackRuntime } from "../../../../command-internal/stack-runtime.ts"; import { loadStackConfig } from "../../../../command-internal/stack-config.ts"; import { withProjectFunctionsEnv } from "../../../../command-internal/stack-functions-env.ts"; import { bold, gray, green, red, yellow } from "../../../../command-internal/colors.ts"; @@ -57,7 +58,7 @@ type StackReport = { readonly project_root: string; readonly branch_context: string; }; - readonly runtime: "native" | "docker"; + readonly runtime: StackRuntime; readonly owner: "reachable" | "unavailable"; readonly lifecycle: Observation["lifecycle"] | null; readonly readiness: "unavailable" | "starting" | "sleeping" | "stopped" | "ready" | "unhealthy"; @@ -148,7 +149,7 @@ const reportFor = ( readonly branchContext: string; readonly stackName: string; }; - readonly runtime: "native" | "docker"; + readonly runtime: StackRuntime; }, owner: StackReport["owner"], observed: ReadonlyArray, diff --git a/apps/cli/tests/helpers/child-process-spawner.ts b/apps/cli/tests/helpers/child-process-spawner.ts index ea6c0e1633..366d65b93a 100644 --- a/apps/cli/tests/helpers/child-process-spawner.ts +++ b/apps/cli/tests/helpers/child-process-spawner.ts @@ -107,10 +107,14 @@ export function mockChildProcessSpawner( export type ContainerEngineState = "missing" | "stopped" | "running"; /** - * Spawner for a host whose `docker` command behaves as `engines` describes. Other commands fail - * with `NotFound`, or run on the real spawner through `hidingLayer`. + * Spawner for a host whose `docker` and `podman` commands behave as `engines` describe, answering + * the identity queries engine target resolution makes. Other commands fail with `NotFound`, or run + * on the real spawner through `hidingLayer`. */ -export function containerEngineSpawner(engines: { readonly docker: ContainerEngineState }) { +export function containerEngineSpawner(engines: { + readonly docker: ContainerEngineState; + readonly podman: ContainerEngineState; +}) { const spawned: SpawnRecord[] = []; const notFound = (command: string) => PlatformError.systemError({ @@ -119,22 +123,31 @@ export function containerEngineSpawner(engines: { readonly docker: ContainerEngi method: "spawn", pathOrDescriptor: command, }); + const answer = (cmd: "docker" | "podman", args: ReadonlyArray) => { + if (cmd === "docker") return args.includes("context") ? "default" : "docker-daemon-id"; + return args.some((arg) => arg.includes("ServiceIsRemote")) + ? "false|true|host|/graph-root" + : "host"; + }; const spawner = (delegate?: ChildProcessSpawner.ChildProcessSpawner["Service"]) => ChildProcessSpawner.make((command) => { const cmd = Predicate.isTagged(command, "StandardCommand") ? command.command : ""; const args = Predicate.isTagged(command, "StandardCommand") ? command.args : []; - if (cmd !== "docker") + if (cmd !== "docker" && cmd !== "podman") return delegate === undefined ? Effect.fail(notFound(cmd)) : delegate.spawn(command); spawned.push({ command: cmd, args }); const state = engines[cmd]; if (state === "missing") return Effect.fail(notFound(cmd)); + const running = state === "running"; return Effect.succeed( ChildProcessSpawner.makeHandle({ pid: ChildProcessSpawner.ProcessId(2000 + spawned.length), - stdout: Stream.empty, - stderr: Stream.empty, + stdout: running ? Stream.make(encoder.encode(answer(cmd, args))) : Stream.empty, + stderr: running + ? Stream.empty + : Stream.make(encoder.encode("Cannot connect to the engine")), all: Stream.empty, - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(state === "running" ? 0 : 1)), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(running ? 0 : 1)), isRunning: Effect.succeed(false), stdin: Sink.drain, kill: () => Effect.void, diff --git a/packages/stack/package.json b/packages/stack/package.json index a5bd12f737..a8ba06429f 100644 --- a/packages/stack/package.json +++ b/packages/stack/package.json @@ -12,6 +12,7 @@ "./internal/dispatch": "./src/internal/dispatch.ts", "./internal/artifacts": "./src/internal/artifacts.ts", "./internal/composition": "./src/internal/composition.ts", + "./internal/engine": "./src/internal/engine.ts", "./internal/release": "./src/internal/release.ts", "./internal/test-run-label": "./src/internal/test-run-label.ts" }, diff --git a/packages/stack/src/internal/engine.ts b/packages/stack/src/internal/engine.ts new file mode 100644 index 0000000000..1bdc19a370 --- /dev/null +++ b/packages/stack/src/internal/engine.ts @@ -0,0 +1,2 @@ +/** Container engine target resolution shared with the CLI's runtime selection. */ +export { resolveEngineTarget, type ContainerEngine } from "../runtime/Container.ts"; From 867e640b7a918f811e97cc206ea28a0db9746610 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 13:25:06 +0200 Subject: [PATCH 090/142] test(stack): run the real-engine stack suites against Podman in CI --- .github/workflows/test.yml | 93 +- .../stack/src/Commands.integration.test.ts | 19 +- packages/stack/src/Owner.integration.test.ts | 183 +- ...ost.container-shutdown.integration.test.ts | 49 +- packages/stack/src/Sweep.integration.test.ts | 62 +- packages/stack/src/effect.integration.test.ts | 6 +- .../stack/src/namespace/Namespace.e2e.test.ts | 19 +- .../stack/src/port-reservation.e2e.test.ts | 6 +- .../src/runtime/Container.integration.test.ts | 133 +- packages/stack/src/runtime/Container.ts | 2 +- ...nalyticsVectorImgproxy.integration.test.ts | 6 +- .../services/AuthStorage.integration.test.ts | 6 +- .../src/services/Database.integration.test.ts | 154 +- .../DatabaseSnapshot.integration.test.ts | 6 +- .../services/Functions.integration.test.ts | 20 +- .../src/services/Mail.integration.test.ts | 6 +- .../src/services/Pooler.integration.test.ts | 8 +- .../src/services/Realtime.integration.test.ts | 6 +- .../src/services/Rest.integration.test.ts | 10 +- .../StorageGateway.integration.test.ts | 8 +- .../src/services/Vector.integration.test.ts | 15 +- .../DockerDatabaseStorage.integration.test.ts | 2528 +++++++++-------- ...ole-stack.podman.idle-parallel.e2e.test.ts | 15 + .../whole-stack.podman.lifecycle.e2e.test.ts | 17 + packages/stack/tests/docker-cleanup.ts | 7 +- .../tests/docker-fixture.integration.test.ts | 176 +- packages/stack/tests/docker-fixture.ts | 10 +- .../docker-volume-run.integration.test.ts | 16 +- packages/stack/tests/docker-volume-run.ts | 16 +- packages/stack/tests/engine-events.ts | 110 + packages/stack/tests/engine-target.ts | 11 +- .../stack/tests/gated-docker-stop-fixture.ts | 2 +- packages/stack/tests/test-engine.ts | 12 + .../stack/tests/whole-stack/confinement.ts | 77 +- packages/stack/tests/whole-stack/fixture.ts | 12 +- packages/stack/tests/whole-stack/scenarios.ts | 49 +- packages/stack/tests/whole-stack/workloads.ts | 6 +- 37 files changed, 2094 insertions(+), 1787 deletions(-) create mode 100644 packages/stack/src/whole-stack.podman.idle-parallel.e2e.test.ts create mode 100644 packages/stack/src/whole-stack.podman.lifecycle.e2e.test.ts create mode 100644 packages/stack/tests/engine-events.ts create mode 100644 packages/stack/tests/test-engine.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3460543918..403627f486 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -326,8 +326,10 @@ jobs: strategy: fail-fast: false matrix: - runtime: [native, docker] + runtime: [native, docker, podman] suite: [lifecycle, idle-parallel] + env: + SUPABASE_STACK_TEST_ENGINE: ${{ matrix.runtime == 'podman' && 'podman' || 'docker' }} steps: - name: Checkout uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1.0.0-beta @@ -338,6 +340,24 @@ jobs: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} go-cache: "false" + # Rootless Podman needs subordinate id ranges and a per-user runtime directory; the user + # service is started explicitly so /run/user/ exists before the first podman call. + - name: Prepare rootless Podman + if: matrix.runtime == 'podman' + run: | + if ! command -v podman >/dev/null; then + sudo apt-get update + sudo apt-get install -y podman + fi + user="$(id -un)" + grep -q "^${user}:" /etc/subuid || sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 "${user}" + sudo loginctl enable-linger "${user}" + sudo systemctl start "user@$(id -u).service" + echo "XDG_RUNTIME_DIR=/run/user/$(id -u)" >> "$GITHUB_ENV" + XDG_RUNTIME_DIR="/run/user/$(id -u)" podman system migrate + podman version + podman info + - name: Run stack end-to-end tests run: >- pnpm --filter @supabase/stack test:e2e:run @@ -351,6 +371,68 @@ jobs: src/public.e2e.test.ts --passWithNoTests=false + test-stack-podman-integration: + if: | + !startsWith(github.head_ref, 'release-notes/') && + (github.event_name == 'merge_group' || + inputs.force || + github.event.pull_request.draft == false) + name: Run stack Podman integration tests + runs-on: blacksmith-8vcpu-ubuntu-2404 + env: + SUPABASE_STACK_TEST_ENGINE: podman + steps: + - name: Checkout + uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1.0.0-beta + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + go-cache: "false" + + # Rootless Podman needs subordinate id ranges and a per-user runtime directory; the user + # service is started explicitly so /run/user/ exists before the first podman call. + - name: Prepare rootless Podman + run: | + if ! command -v podman >/dev/null; then + sudo apt-get update + sudo apt-get install -y podman + fi + user="$(id -un)" + grep -q "^${user}:" /etc/subuid || sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 "${user}" + sudo loginctl enable-linger "${user}" + sudo systemctl start "user@$(id -u).service" + echo "XDG_RUNTIME_DIR=/run/user/$(id -u)" >> "$GITHUB_ENV" + XDG_RUNTIME_DIR="/run/user/$(id -u)" podman system migrate + podman version + podman info + + # The suites that drive a real container engine; the rest of the integration project is + # engine-independent and runs in test-integration. + - name: Run stack Podman integration tests + working-directory: packages/stack + run: >- + pnpm test:integration:run + src/Owner.integration.test.ts + src/Sweep.integration.test.ts + src/Commands.integration.test.ts + src/StackHost.container-shutdown.integration.test.ts + src/runtime/Container.integration.test.ts + src/storage/DockerDatabaseStorage.integration.test.ts + src/services/Database.integration.test.ts + src/services/Functions.integration.test.ts + src/services/Realtime.integration.test.ts + src/services/Rest.integration.test.ts + src/services/AuthStorage.integration.test.ts + src/services/StorageGateway.integration.test.ts + src/services/AnalyticsVectorImgproxy.integration.test.ts + src/services/Mail.integration.test.ts + src/services/Pooler.integration.test.ts + src/services/Vector.integration.test.ts + tests/docker-volume-run.integration.test.ts + --passWithNoTests=false + # Summary job that gates branch protection. It preserves the original check # name and succeeds iff every CLI shard and stack runtime succeeded. test-e2e-summary: @@ -361,15 +443,16 @@ jobs: inputs.force || github.event.pull_request.draft == false) name: Run end-to-end tests - needs: [test-e2e, test-stack-e2e, test-stack-native-scenarios] + needs: [test-e2e, test-stack-e2e, test-stack-native-scenarios, test-stack-podman-integration] runs-on: ubuntu-latest steps: - name: Verify all shards succeeded run: | if [ "${{ needs.test-e2e.result }}" = "failure" ] || [ "${{ needs.test-e2e.result }}" = "cancelled" ] || \ [ "${{ needs.test-stack-e2e.result }}" = "failure" ] || [ "${{ needs.test-stack-e2e.result }}" = "cancelled" ] || \ - [ "${{ needs.test-stack-native-scenarios.result }}" = "failure" ] || [ "${{ needs.test-stack-native-scenarios.result }}" = "cancelled" ]; then - echo "::error ::One or more e2e jobs failed: cli=${{ needs.test-e2e.result }}, stack=${{ needs.test-stack-e2e.result }}, stack-native-scenarios=${{ needs.test-stack-native-scenarios.result }}" + [ "${{ needs.test-stack-native-scenarios.result }}" = "failure" ] || [ "${{ needs.test-stack-native-scenarios.result }}" = "cancelled" ] || \ + [ "${{ needs.test-stack-podman-integration.result }}" = "failure" ] || [ "${{ needs.test-stack-podman-integration.result }}" = "cancelled" ]; then + echo "::error ::One or more e2e jobs failed: cli=${{ needs.test-e2e.result }}, stack=${{ needs.test-stack-e2e.result }}, stack-native-scenarios=${{ needs.test-stack-native-scenarios.result }}, stack-podman-integration=${{ needs.test-stack-podman-integration.result }}" exit 1 fi - echo "All e2e jobs reported: cli=${{ needs.test-e2e.result }}, stack=${{ needs.test-stack-e2e.result }}, stack-native-scenarios=${{ needs.test-stack-native-scenarios.result }}" + echo "All e2e jobs reported: cli=${{ needs.test-e2e.result }}, stack=${{ needs.test-stack-e2e.result }}, stack-native-scenarios=${{ needs.test-stack-native-scenarios.result }}, stack-podman-integration=${{ needs.test-stack-podman-integration.result }}" diff --git a/packages/stack/src/Commands.integration.test.ts b/packages/stack/src/Commands.integration.test.ts index f6f61eb37a..c27b95b6f0 100644 --- a/packages/stack/src/Commands.integration.test.ts +++ b/packages/stack/src/Commands.integration.test.ts @@ -18,6 +18,7 @@ import { import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; import { postgres } from "./Commands.ts"; +import type { ContainerEngine } from "./runtime/Container.ts"; import * as CommandRunner from "./host/CommandRunner.ts"; import { CommandError } from "./host/CommandRunner.ts"; import type { PgProveOptions, PostgresCommand } from "./Commands.ts"; @@ -25,7 +26,7 @@ import { makeDatabase } from "./services/Database.ts"; import { makeStandaloneService } from "../tests/standalone-service.ts"; import { bindTcp, serveTcp } from "./Proxy.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; -import { dockerEngineTarget } from "../tests/engine-target.ts"; +import { engineTarget, testEngine } from "../tests/engine-target.ts"; const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; @@ -37,12 +38,12 @@ const makeTestCommandRunner = (options: { readonly stackId: string; readonly root: string; readonly cacheRoot: string; - readonly runtime: "native" | "docker"; + readonly runtime: "native" | ContainerEngine; }) => Layer.build( CommandRunner.layer({ ...options, - ...(options.runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), + ...(options.runtime !== "native" ? { engineTarget } : {}), }), ).pipe(Effect.map((context) => Context.get(context, CommandRunner.Service))); @@ -77,14 +78,14 @@ const runPostgres = ( }); describe("finite PostgreSQL commands", { timeout: 180_000 }, () => { - for (const runtime of ["native", "docker"] as const) { + for (const runtime of ["native", testEngine] as const) { it.live(`${runtime} runs SQL from stdin, streams a dump, and returns client failure`, () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const stackId = `tools-integration-${randomUUID()}`; const root = - runtime === "docker" + runtime !== "native" ? yield* makeDockerDatabaseRoot("stack-tools-", stackId) : yield* fs.makeTempDirectoryScoped({ prefix: "stack-tools-" }); const database = yield* makeDatabase({ @@ -93,7 +94,7 @@ describe("finite PostgreSQL commands", { timeout: 180_000 }, () => { stackId, instanceId: "database", runtime, - ...(runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), + ...(runtime !== "native" ? { engineTarget } : {}), }); const service = yield* makeStandaloneService(database.definition, { id: "database", @@ -228,14 +229,14 @@ describe("finite PostgreSQL commands", { timeout: 180_000 }, () => { } for (const major of [15, 17] as const) { - for (const runtime of ["native", "docker"] as const) { + for (const runtime of ["native", testEngine] as const) { it.live(`${runtime} runs bundled pg_prove for PostgreSQL ${major}`, () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const stackId = `tools-pgprove-${runtime}-${major}-${randomUUID()}`; const root = - runtime === "docker" + runtime !== "native" ? yield* makeDockerDatabaseRoot(`stack-pgprove-${major}-`, stackId) : yield* fs.makeTempDirectoryScoped({ prefix: `stack-pgprove-${major}-` }); const database = yield* makeDatabase({ @@ -244,7 +245,7 @@ describe("finite PostgreSQL commands", { timeout: 180_000 }, () => { stackId, instanceId: "database", runtime, - ...(runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), + ...(runtime !== "native" ? { engineTarget } : {}), }); const service = yield* makeStandaloneService(database.definition, { id: "database", diff --git a/packages/stack/src/Owner.integration.test.ts b/packages/stack/src/Owner.integration.test.ts index 0f2ac45c51..ce5a8ea389 100644 --- a/packages/stack/src/Owner.integration.test.ts +++ b/packages/stack/src/Owner.integration.test.ts @@ -1,6 +1,6 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { PgClient } from "@effect/sql-pg"; -import { expect, it } from "@effect/vitest"; +import { describe, expect, it } from "@effect/vitest"; import { Context, Effect, FileSystem, Layer, Redacted, Ref, Schema, Stream } from "effect"; import { HttpClient } from "effect/unstable/http"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; @@ -15,7 +15,7 @@ import type { SavedStack } from "./StackNamespace.ts"; import { DEFAULT_LOCAL_JWT_SECRET } from "./Defaults.ts"; import type { ServiceCreationInput } from "./services/Catalog.ts"; import { ownerFor } from "../tests/owner-rpc.ts"; -import { dockerEngineTarget } from "../tests/engine-target.ts"; +import { engineTarget, testEngine } from "../tests/engine-target.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; const stateFor = (root: string) => @@ -35,11 +35,10 @@ const initial = (id: string): SavedStack => ({ composition: { members: [], dependencies: [] }, }); -const initialDocker = (id: string): SavedStack => ({ ...initial(id), runtime: "docker" }); +const initialContainer = (id: string): SavedStack => ({ ...initial(id), runtime: testEngine }); -/** Container ids docker still reports for `stackId`, by the label every instance container carries. */ /** - * Container ids docker still reports for `instanceId`, by the labels an instance's own containers + * Container ids the engine still reports for `instanceId`, by the labels an instance's own containers * carry. Excludes a stack-wide shared volume helper (labeled only with the stack, never the * instance), which outlives any one instance's destroy and is torn down on its own lifecycle. */ @@ -49,8 +48,9 @@ const containersForInstance = (stackId: string, instanceId: string) => const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( ChildProcess.make( - "docker", + testEngine, [ + ...engineTarget.argv, "ps", "-a", "--filter", @@ -981,7 +981,7 @@ it.live( const tempRoot = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-no-registration-state-", }); - const stack = initialDocker(stackId); + const stack = initialContainer(stackId); const state = yield* stateFor(`${tempRoot}/state`); yield* state.save(stack); const owner = yield* ownerFor({ @@ -989,7 +989,7 @@ it.live( state, root: dataRoot, cacheRoot, - engineTarget: dockerEngineTarget, + engineTarget, }); yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); @@ -1040,92 +1040,95 @@ it.live( 120_000, ); -it.live( - "destroys a database's container and storage namespace, and releases its port reservation, after the whole data root is deleted", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const stackId = `owner-no-data-root-${randomUUID().slice(0, 8)}`; - const dataRoot = yield* makeDockerDatabaseRoot("stack-owner-no-data-root-", stackId); - const tempRoot = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-owner-no-data-root-state-", - }); - const stack = initialDocker(stackId); - const state = yield* stateFor(`${tempRoot}/state`); - yield* state.save(stack); - const owner = yield* ownerFor({ - saved: stack, - state, - root: dataRoot, - cacheRoot, - engineTarget: dockerEngineTarget, - }); - yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); +// The Docker volume backend has no Podman counterpart: Podman keeps database data in host directories. +describe.runIf(testEngine === "docker")("Docker volume storage", () => { + it.live( + "destroys a database's container and storage namespace, and releases its port reservation, after the whole data root is deleted", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const stackId = `owner-no-data-root-${randomUUID().slice(0, 8)}`; + const dataRoot = yield* makeDockerDatabaseRoot("stack-owner-no-data-root-", stackId); + const tempRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-owner-no-data-root-state-", + }); + const stack = initialContainer(stackId); + const state = yield* stateFor(`${tempRoot}/state`); + yield* state.save(stack); + const owner = yield* ownerFor({ + saved: stack, + state, + root: dataRoot, + cacheRoot, + engineTarget, + }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); - const created = yield* owner.rpc.createService({ - service: "database", - config: { - version: "17", - databasePassword: Redacted.make("owner-no-data-root-password"), - jwtSecret: Redacted.make("owner-no-data-root-jwt-secret-at-least-32-characters"), - jwtExpiry: 3600, - }, - endpoints: { sql: { port: "auto" } }, - }); - yield* owner.rpc.startService({ id: created.id }); - yield* owner.rpc.readyService({ id: created.id }); + const created = yield* owner.rpc.createService({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("owner-no-data-root-password"), + jwtSecret: Redacted.make("owner-no-data-root-jwt-secret-at-least-32-characters"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* owner.rpc.startService({ id: created.id }); + yield* owner.rpc.readyService({ id: created.id }); - const realStateRoot = yield* fs.realPath(`${tempRoot}/state`); - const portReservations = Context.get( - yield* Layer.build(PortReservations.layer), - PortReservations.Service, - ); - expect( - yield* portReservations.find(realStateRoot, stackId, `${created.id}:sql`), - ).toBeDefined(); - expect(yield* containersForInstance(stackId, created.id)).not.toHaveLength(0); - const marker = yield* Schema.decodeEffect(Schema.fromJsonString(InstanceMarker))( - yield* fs.readFileString(`${dataRoot}/${created.id}/.supabase-database-storage.json`), - ); - if (marker.backend !== "docker" || marker.volume === undefined) - return yield* Effect.die("Docker test selected host fallback"); - const volume = marker.volume; + const realStateRoot = yield* fs.realPath(`${tempRoot}/state`); + const portReservations = Context.get( + yield* Layer.build(PortReservations.layer), + PortReservations.Service, + ); + expect( + yield* portReservations.find(realStateRoot, stackId, `${created.id}:sql`), + ).toBeDefined(); + expect(yield* containersForInstance(stackId, created.id)).not.toHaveLength(0); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(InstanceMarker))( + yield* fs.readFileString(`${dataRoot}/${created.id}/.supabase-database-storage.json`), + ); + if (marker.backend !== "docker" || marker.volume === undefined) + return yield* Effect.die("Docker test selected host fallback"); + const volume = marker.volume; - // The database sleeps; its storage object (inside the still-running owner) keeps the - // identity it already resolved in memory, then the whole owned data root disappears (the - // volume host, a disk failure, manual cleanup). - yield* owner.rpc.stopService({ id: created.id }); - expect(yield* containersForInstance(stackId, created.id)).toHaveLength(0); - yield* fs.remove(dataRoot, { recursive: true, force: true }); + // The database sleeps; its storage object (inside the still-running owner) keeps the + // identity it already resolved in memory, then the whole owned data root disappears (the + // volume host, a disk failure, manual cleanup). + yield* owner.rpc.stopService({ id: created.id }); + expect(yield* containersForInstance(stackId, created.id)).toHaveLength(0); + yield* fs.remove(dataRoot, { recursive: true, force: true }); - yield* owner.rpc.destroyService({ id: created.id }); + yield* owner.rpc.destroyService({ id: created.id }); - expect(yield* containersForInstance(stackId, created.id)).toHaveLength(0); - expect( - yield* portReservations.find(realStateRoot, stackId, `${created.id}:sql`), - ).toBeUndefined(); - const remainingNamespace = yield* runDocker([ - "run", - "--rm", - "--mount", - `type=volume,src=${volume},dst=/store`, - "busybox:1.36", - "/bin/sh", - "-c", - `[ -e /store/${marker.namespace} ] && echo present || echo absent`, - ]); - expect(remainingNamespace.trim()).toBe("absent"); - yield* runDocker(["volume", "rm", volume]); - }), - ).pipe( - Effect.provide( - Layer.mergeAll( - NodeServices.layer, - NodeHttpClient.layerNodeHttp, - PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + expect(yield* containersForInstance(stackId, created.id)).toHaveLength(0); + expect( + yield* portReservations.find(realStateRoot, stackId, `${created.id}:sql`), + ).toBeUndefined(); + const remainingNamespace = yield* runDocker([ + "run", + "--rm", + "--mount", + `type=volume,src=${volume},dst=/store`, + "docker.io/library/busybox:1.36", + "/bin/sh", + "-c", + `[ -e /store/${marker.namespace} ] && echo present || echo absent`, + ]); + expect(remainingNamespace.trim()).toBe("absent"); + yield* runDocker(["volume", "rm", volume]); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), ), ), - ), - 120_000, -); + 120_000, + ); +}); diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index 53aceeeb3e..2730dc21b5 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -26,8 +26,9 @@ import { waitForOwnerExit, type HostAccess, } from "./HostProcess.ts"; -import { makeContainerRuntime, resolveEngineTarget } from "./runtime/Container.ts"; +import { makeContainerRuntime } from "./runtime/Container.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; +import { engineTarget, testEngine } from "../tests/engine-target.ts"; import { holdReleaseFifo } from "../tests/release-fifo.ts"; import { shutdownOwner, watchLeaseRelease } from "../tests/owner.ts"; import { watchEntry } from "../tests/watch-entry.ts"; @@ -42,12 +43,16 @@ const gatedDockerStopFixture = fileURLToPath( const helperImage = "public.ecr.aws/docker/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; -const docker = Effect.fn("StackHostContainerShutdownTest.docker")((args: ReadonlyArray) => +const engine = Effect.fn("StackHostContainerShutdownTest.engine")((args: ReadonlyArray) => Effect.scoped( Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", args, { stdin: "ignore", stdout: "pipe", stderr: "pipe" }), + ChildProcess.make(testEngine, [...engineTarget.argv, ...args], { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }), ); const [stdout, stderr, code] = yield* Effect.all( [ @@ -58,7 +63,7 @@ const docker = Effect.fn("StackHostContainerShutdownTest.docker")((args: Readonl { concurrency: "unbounded" }, ); if (Number(code) !== 0) - return yield* Effect.die(`docker ${args.join(" ")} failed: ${stderr}`); + return yield* Effect.die(`${testEngine} ${args.join(" ")} failed: ${stderr}`); return stdout.trim(); }), ), @@ -70,7 +75,7 @@ const stateFor = (root: string) => ); const containers = (stackId: string, dataRoot: string) => - docker([ + engine([ "ps", "--all", "--quiet", @@ -84,7 +89,7 @@ const containers = (stackId: string, dataRoot: string) => const removeContainers = (stackId: string, dataRoot: string) => containers(stackId, dataRoot).pipe( Effect.flatMap((ids) => - Effect.forEach(ids, (id) => docker(["rm", "--force", id]).pipe(Effect.ignore), { + Effect.forEach(ids, (id) => engine(["rm", "--force", id]).pipe(Effect.ignore), { concurrency: 1, discard: true, }), @@ -98,7 +103,7 @@ const startHost = (stateRoot: string, cacheRoot: string, stackId: string, projec if (current === undefined) yield* state.save({ id: stackId, - runtime: "docker", + runtime: testEngine, identity: { projectRoot, branchContext: "container-shutdown-test", stackName: stackId }, instances: [], lifetime: "detached", @@ -126,10 +131,8 @@ it.live.skipIf(process.platform === "win32")( const rootA = path.dirname(path.dirname(dataA)); const rootB = path.dirname(path.dirname(dataB)); const cacheRoot = `${base}/cache`; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const target = yield* resolveEngineTarget(spawner); const helper = yield* makeContainerRuntime({ - target, + target: engineTarget, root: dataA, }); yield* helper.prepare(helperImage); @@ -251,7 +254,7 @@ it.live.skipIf(process.platform === "win32")( const stateB3 = yield* stateFor(rootB); yield* stateA3.save({ id: stackId, - runtime: "docker", + runtime: testEngine, identity: { projectRoot: `${base}/project-a`, branchContext: "container-shutdown-test", @@ -263,7 +266,7 @@ it.live.skipIf(process.platform === "win32")( }); yield* stateB3.save({ id: stackId, - runtime: "docker", + runtime: testEngine, identity: { projectRoot: `${base}/project-b`, branchContext: "container-shutdown-test", @@ -360,7 +363,7 @@ it.live.skipIf(process.platform === "win32")( const state = yield* stateFor(stateRoot); yield* state.save({ id: stackId, - runtime: "docker", + runtime: testEngine, identity: { projectRoot: `${base}/project`, branchContext: "abandon-docker-test", @@ -426,9 +429,9 @@ it.live.skipIf(process.platform === "win32")( ); /** - * Installs a `docker` shim the owner finds first on its PATH: the first `docker stop` announces + * Installs an engine CLI shim the owner finds first on its PATH: the first `stop` announces * itself by creating `waiting` in `gateDir` and blocks until the test writes the `release` FIFO, - * then every command runs the real docker. Returns the idempotent, scope-owned `release`. + * then every command runs the real engine. Returns the idempotent, scope-owned `release`. */ const holdDockerStops = (gateDir: string) => Effect.gen(function* () { @@ -440,7 +443,7 @@ const holdDockerStops = (gateDir: string) => Effect.scoped, ); yield* fs.writeFileString( - `${gateDir}/bin/docker`, + `${gateDir}/bin/${testEngine}`, [ "#!/bin/sh", `gate='${gateDir}'`, @@ -455,7 +458,7 @@ const holdDockerStops = (gateDir: string) => " done", "fi", 'PATH="${PATH#"$gate/bin:"}"', - 'exec docker "$@"', + `exec ${testEngine} "$@"`, "", ].join("\n"), { mode: 0o755 }, @@ -493,7 +496,7 @@ const abandonsWhileStopRuns = (trigger: StopTrigger) => const state = yield* stateFor(stateRoot); yield* state.save({ id: stackId, - runtime: "docker", + runtime: testEngine, identity: { projectRoot: `${base}/project`, branchContext: "abandon-gated-test", @@ -545,7 +548,7 @@ const abandonsWhileStopRuns = (trigger: StopTrigger) => "the stack's shared container-env scratch directory exists", ).toBe(true); - // Subscribes to the held `docker stop`'s own marker before triggering the stop, so the + // Subscribes to the held engine stop's own marker before triggering the stop, so the // registration deletion below never races the gate itself. const waiting = yield* watchEntry(gateDir, "waiting", true); yield* triggerStop(trigger, access); @@ -602,7 +605,7 @@ it.live.skipIf(process.platform === "win32")( const state = yield* stateFor(stateRoot); yield* state.save({ id: stackId, - runtime: "docker", + runtime: testEngine, identity: { projectRoot: `${base}/project`, branchContext: "abandon-docker-label-test", @@ -623,7 +626,7 @@ it.live.skipIf(process.platform === "win32")( // container carries only the stack's identity label, simulating a leaked storage helper // no claim or helper-registry bookkeeping ever reaches. Only the // registration-independent label sweep can remove it. - yield* docker([ + yield* engine([ "run", "-d", "--name", @@ -679,7 +682,7 @@ it.live.skipIf(process.platform === "win32")( const state = yield* stateFor(stateRoot); yield* state.save({ id: stackId, - runtime: "docker", + runtime: testEngine, identity: { projectRoot: `${base}/project`, branchContext: "abandon-docker-root-test", @@ -796,7 +799,7 @@ it.live.skipIf(process.platform === "win32")( const state = yield* stateFor(stateRoot); yield* state.save({ id: stackId, - runtime: "docker", + runtime: testEngine, identity: { projectRoot: `${base}/project`, branchContext: "pinned-postgres-test", diff --git a/packages/stack/src/Sweep.integration.test.ts b/packages/stack/src/Sweep.integration.test.ts index 567bef4a67..a54c976e3d 100644 --- a/packages/stack/src/Sweep.integration.test.ts +++ b/packages/stack/src/Sweep.integration.test.ts @@ -9,7 +9,6 @@ import { Fiber, FileSystem, Layer, - Option, Path, Queue, Stream, @@ -18,10 +17,12 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- test-only native watcher for synchronous attachment, no Effect wrapper gives this guarantee. import { watch as nodeWatch } from "node:fs"; import { launchHost } from "./HostProcess.ts"; -import { makeContainerRuntime, resolveEngineTarget } from "./runtime/Container.ts"; +import { makeContainerRuntime } from "./runtime/Container.ts"; import * as StackNamespace from "./StackNamespace.ts"; import { sweepOrphans } from "./Sweep.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; +import { awaitContainerRemoved } from "../tests/engine-events.ts"; +import { engineTarget, testEngine } from "../tests/engine-target.ts"; import { shutdownOwner, watchLeaseRelease } from "../tests/owner.ts"; class SweepTestError extends Data.TaggedError("SweepTestError")<{ readonly message: string }> {} @@ -29,12 +30,16 @@ class SweepTestError extends Data.TaggedError("SweepTestError")<{ readonly messa const helperImage = "public.ecr.aws/docker/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; -const docker = Effect.fn("SweepTest.docker")((args: ReadonlyArray) => +const engine = Effect.fn("SweepTest.engine")((args: ReadonlyArray) => Effect.scoped( Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", args, { stdin: "ignore", stdout: "pipe", stderr: "pipe" }), + ChildProcess.make(testEngine, [...engineTarget.argv, ...args], { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }), ); const [stdout, stderr, code] = yield* Effect.all( [ @@ -45,7 +50,7 @@ const docker = Effect.fn("SweepTest.docker")((args: ReadonlyArray) => { concurrency: "unbounded" }, ); if (Number(code) !== 0) - return yield* Effect.die(`docker ${args.join(" ")} failed: ${stderr}`); + return yield* Effect.die(`${testEngine} ${args.join(" ")} failed: ${stderr}`); return stdout.trim(); }), ), @@ -57,7 +62,7 @@ const labels = (stackId: string, dataRoot: string) => [ ]; const containers = (stackId: string, dataRoot: string) => - docker([ + engine([ "ps", "--all", "--quiet", @@ -67,45 +72,12 @@ const containers = (stackId: string, dataRoot: string) => const createOwnedContainer = (stackId: string, dataRoot: string) => Effect.acquireRelease( - docker([ + engine([ "create", ...labels(stackId, dataRoot).flatMap((label) => ["--label", label.slice("label=".length)]), helperImage, ]), - (id) => docker(["rm", "--force", id]).pipe(Effect.ignore), - ); - -/** Completes when Docker reports the container destroyed, including events before subscription. */ -const awaitDestroyed = (id: string, since: number) => - Effect.scoped( - Effect.gen(function* () { - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const events = yield* spawner.spawn( - ChildProcess.make( - "docker", - [ - "events", - "--since", - String(since), - "--filter", - `container=${id}`, - "--filter", - "event=destroy", - "--format", - "{{.Actor.ID}}", - ], - { stdin: "ignore", stdout: "pipe", stderr: "ignore" }, - ), - ); - const destroyed = yield* events.stdout.pipe( - Stream.decodeText, - Stream.splitLines, - Stream.filter((line) => line.trim() === id), - Stream.runHead, - ); - if (Option.isNone(destroyed)) - return yield* new SweepTestError({ message: "Docker event stream ended" }); - }), + (id) => engine(["rm", "--force", id]).pipe(Effect.ignore), ); /** @@ -170,10 +142,8 @@ it.live.skipIf(process.platform === "win32")( ); const rootA = path.dirname(path.dirname(dataA)); const cacheRoot = `${path.dirname(rootA)}/cache`; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const target = yield* resolveEngineTarget(spawner); const helper = yield* makeContainerRuntime({ - target, + target: engineTarget, root: dataA, }); yield* helper.prepare(helperImage); @@ -182,7 +152,7 @@ it.live.skipIf(process.platform === "win32")( StackNamespace.Service, ); - yield* state.save(saved(deadId, `${rootA}/dead`, "docker", "detached")); + yield* state.save(saved(deadId, `${rootA}/dead`, testEngine, "detached")); const dead = (yield* launchHost(state, { stateRoot: rootA, cacheRoot, stackId: deadId })) .endpoint; const orphan = yield* createOwnedContainer(deadId, dataA); @@ -200,7 +170,7 @@ it.live.skipIf(process.platform === "win32")( const since = Math.floor((yield* Clock.currentTimeMillis) / 1000) - 1; const swept = yield* Effect.all( [ - awaitDestroyed(orphan, since), + awaitContainerRemoved(orphan, since), awaitRemoval(path.join(rootA, sessionId), "state.json"), awaitRemoval(path.join(rootA, deadId), "owner.json"), ], diff --git a/packages/stack/src/effect.integration.test.ts b/packages/stack/src/effect.integration.test.ts index d67837ed70..34423e1401 100644 --- a/packages/stack/src/effect.integration.test.ts +++ b/packages/stack/src/effect.integration.test.ts @@ -25,6 +25,8 @@ import { } from "./effect.ts"; import { initialization, postgres } from "./Commands.ts"; import { fileURLToPath } from "node:url"; +import { testEngine } from "../tests/test-engine.ts"; +import type { ContainerEngine } from "./runtime/Container.ts"; import { launchHost } from "./HostProcess.ts"; import * as PromiseApi from "./index.ts"; import * as StackNamespace from "./StackNamespace.ts"; @@ -264,7 +266,7 @@ it.live( { timeout: 60_000 }, ); -const resetDataStory = (runtime: "native" | "docker") => +const resetDataStory = (runtime: "native" | ContainerEngine) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: `stack-reset-data-${runtime}-` }); @@ -445,7 +447,7 @@ it.live("resets native database data through the public RPC", () => resetDataSto timeout: 10 * 60_000, }); -it.live("resets Docker database data through the public RPC", () => resetDataStory("docker"), { +it.live("resets container database data through the public RPC", () => resetDataStory(testEngine), { timeout: 15 * 60_000, }); diff --git a/packages/stack/src/namespace/Namespace.e2e.test.ts b/packages/stack/src/namespace/Namespace.e2e.test.ts index d55859c5cd..910719dd37 100644 --- a/packages/stack/src/namespace/Namespace.e2e.test.ts +++ b/packages/stack/src/namespace/Namespace.e2e.test.ts @@ -25,6 +25,7 @@ import { import { create, open } from "../effect.ts"; import * as StackNamespace from "../StackNamespace.ts"; import { captureOwnerPid, watchLeaseRelease } from "../../tests/owner.ts"; +import { testEngine } from "../../tests/test-engine.ts"; class PublishBarrierError extends Data.TaggedError("PublishBarrierError")<{ readonly message: string; @@ -317,12 +318,12 @@ it.live( 20_000, ); -const docker = Effect.fn("Namespace.e2e.docker")((args: ReadonlyArray) => +const engine = Effect.fn("Namespace.e2e.engine")((args: ReadonlyArray) => Effect.scoped( Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", args, { stdin: "ignore", stdout: "pipe", stderr: "pipe" }), + ChildProcess.make(testEngine, args, { stdin: "ignore", stdout: "pipe", stderr: "pipe" }), ); const [stdout, stderr, code] = yield* Effect.all( [ @@ -333,13 +334,13 @@ const docker = Effect.fn("Namespace.e2e.docker")((args: ReadonlyArray) = { concurrency: "unbounded" }, ); if (Number(code) !== 0) - return yield* Effect.die(`docker ${args.join(" ")} failed: ${stderr}`); + return yield* Effect.die(`${testEngine} ${args.join(" ")} failed: ${stderr}`); return stdout.trim(); }), ), ); -// Needs a real Docker daemon, which only the Linux CI runner has. +// Needs a real container engine, which only the Linux CI runner has. it.live.skipIf(process.platform === "win32" || process.platform === "darwin")( "removes exactly the containers a SIGKILLed owner left behind, with no manual cleanup", () => @@ -353,7 +354,7 @@ it.live.skipIf(process.platform === "win32" || process.platform === "darwin")( projectRoot: root, stateRoot, cacheRoot, - runtime: "docker", + runtime: testEngine, startOwner: true, }); yield* Effect.addFinalizer(() => stack.destroy.pipe(Effect.ignore)); @@ -361,7 +362,7 @@ it.live.skipIf(process.platform === "win32" || process.platform === "darwin")( yield* mail.start; yield* mail.ready; - const containerIds = (yield* docker([ + const containerIds = (yield* engine([ "ps", "--all", "--quiet", @@ -383,7 +384,7 @@ it.live.skipIf(process.platform === "win32" || process.platform === "darwin")( // Killing the owner leaves the container running; nothing in this test removes it. expect( - yield* docker(["inspect", "--format", "{{.State.Status}}", String(containerId)]), + yield* engine(["inspect", "--format", "{{.State.Status}}", String(containerId)]), ).not.toBe(""); const reopened = yield* open({ @@ -394,7 +395,7 @@ it.live.skipIf(process.platform === "win32" || process.platform === "darwin")( }); yield* Effect.addFinalizer(() => reopened.destroy.pipe(Effect.ignore)); - const remaining = yield* docker([ + const remaining = yield* engine([ "ps", "--all", "--quiet", @@ -403,7 +404,7 @@ it.live.skipIf(process.platform === "win32" || process.platform === "darwin")( `label=com.supabase.stack=${stack.id}`, ]); expect(remaining).toBe(""); - const inspectExit = yield* docker([ + const inspectExit = yield* engine([ "inspect", "--format", "{{.State.Status}}", diff --git a/packages/stack/src/port-reservation.e2e.test.ts b/packages/stack/src/port-reservation.e2e.test.ts index 3498ed077f..12e3f10627 100644 --- a/packages/stack/src/port-reservation.e2e.test.ts +++ b/packages/stack/src/port-reservation.e2e.test.ts @@ -7,6 +7,8 @@ import { ownerExitProbe, waitForOwnerExit } from "./HostProcess.ts"; import * as PortReservations from "./namespace/PortReservations.ts"; import { captureOwnerPid } from "../tests/owner.ts"; import { destroyTestStack } from "../tests/stack-cleanup.ts"; +import { testEngine } from "../tests/test-engine.ts"; +import type { ContainerEngine } from "./runtime/Container.ts"; /** * Step 1.4's per-user port reservation registry: a stopped stack keeps its public ports across @@ -44,7 +46,7 @@ const foreignListener = (host: string, port: number) => const makeStack = Effect.fn("PortReservation.makeStack")(function* ( fs: FileSystem.FileSystem, cacheRoot: string, - runtime: "native" | "docker" = "native", + runtime: "native" | ContainerEngine = "native", ) { const root = yield* fs.makeTempDirectoryScoped({ prefix: `port-reservation-${runtime}-` }); const stack = yield* create({ @@ -281,7 +283,7 @@ it.live( yield* stackA.destroy; const { stack: stackB } = yield* makeStack(fs, cacheRoot, "native"); - const { stack: stackC } = yield* makeStack(fs, cacheRoot, "docker"); + const { stack: stackC } = yield* makeStack(fs, cacheRoot, testEngine); const mailB = yield* stackB.services.create(mailOn(port)); const mailC = yield* stackC.services.create(mailOn(port)); const [exitB, exitC] = yield* Effect.all( diff --git a/packages/stack/src/runtime/Container.integration.test.ts b/packages/stack/src/runtime/Container.integration.test.ts index 1129807967..9317b19c54 100644 --- a/packages/stack/src/runtime/Container.integration.test.ts +++ b/packages/stack/src/runtime/Container.integration.test.ts @@ -35,6 +35,7 @@ import { type ContainerProcess, type EngineTarget, } from "./Container.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; import * as Owner from "../Owner.ts"; import * as StackNamespace from "../StackNamespace.ts"; import { makeDockerDatabaseStorage } from "../storage/DockerDatabaseStorage.ts"; @@ -45,11 +46,11 @@ const image = await Effect.gen(function* () { const path = yield* Path.Path; const file = yield* path.fromFileUrl(new URL("../../../../.bun-version", import.meta.url)); const version = yield* fs.readFileString(file); - return `oven/bun:${version.trim()}-slim`; + return `docker.io/oven/bun:${version.trim()}-slim`; }).pipe(Effect.provide(NodeServices.layer), Effect.runPromise); // An unpinned target: these tests exercise the container runtime, not endpoint pinning, so an // empty argv prefix leaves every command exactly as it was before pinning existed. -const dockerTarget: EngineTarget = { engine: "docker", argv: [], daemonId: "test-daemon-id" }; +const containerTarget: EngineTarget = { ...engineTarget, argv: [] }; const ipv4 = /^(?:\d{1,3}\.){3}\d{1,3}$/u; const stoppableIdleScript = "process.on('SIGTERM', () => process.exit(0)); setInterval(() => {}, 1000)"; @@ -64,7 +65,7 @@ describe("container process adapter", () => { Effect.gen(function* () { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -75,7 +76,7 @@ describe("container process adapter", () => { const pullAttempted = yield* Ref.make(false); const spawner = makePullFailureSpawner(delegate, pullAttempted); yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }).pipe( Effect.flatMap((runtime) => runtime.prepare(pinnedImage)), @@ -93,7 +94,7 @@ describe("container process adapter", () => { const pullAttempted = yield* Ref.make(false); const spawner = makePullFailureSpawner(delegate, pullAttempted); const result = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }).pipe( Effect.flatMap((runtime) => @@ -112,7 +113,7 @@ describe("container process adapter", () => { const id = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -149,7 +150,7 @@ describe("container process adapter", () => { const id = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -188,7 +189,7 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", hostGateway: yield* makeHostGateway, }); @@ -226,7 +227,7 @@ describe("container process adapter", () => { const probes = yield* Ref.make(0); const hostGateway = yield* makeHostGateway; const makeRuntime = makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", hostGateway, }).pipe( @@ -271,13 +272,13 @@ describe("container process adapter", () => { const hostGateway = yield* makeHostGateway; const spawner = makeHostGatewayProbeSpawner(delegate, probes, () => undefined, release); const database = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", hostGateway, awaitHostGateway: false, }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)); const service = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", hostGateway, }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)); @@ -312,7 +313,7 @@ describe("container process adapter", () => { Effect.gen(function* () { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", hostGateway: yield* makeHostGateway, }).pipe( @@ -348,7 +349,7 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", hostGateway: yield* makeHostGateway, }).pipe( @@ -386,7 +387,7 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", hostGateway: yield* makeHostGateway, }).pipe( @@ -425,7 +426,7 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", hostGateway: yield* makeHostGateway, }).pipe( @@ -457,7 +458,7 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", hostGateway: yield* makeHostGateway, }).pipe( @@ -492,7 +493,7 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", hostGateway: yield* makeHostGateway, }).pipe( @@ -531,7 +532,7 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const database = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", hostGateway: yield* makeHostGateway, awaitHostGateway: false, @@ -564,7 +565,7 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -591,7 +592,7 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -617,7 +618,7 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -650,7 +651,7 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -689,7 +690,7 @@ describe("container process adapter", () => { it.live("stops a container with its configured stop signal", () => Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -718,7 +719,7 @@ describe("container process adapter", () => { it.live("waits until a discarded container stops before returning", () => Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -748,7 +749,7 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -779,7 +780,7 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -810,7 +811,7 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const released = yield* Deferred.make(); const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }).pipe( Effect.provideService( @@ -851,7 +852,7 @@ describe("container process adapter", () => { yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }).pipe( Effect.provideService( @@ -884,7 +885,7 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -919,7 +920,7 @@ describe("container process adapter", () => { const result = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -959,7 +960,7 @@ describe("container process adapter", () => { const result = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -1004,7 +1005,7 @@ describe("container process adapter", () => { const result = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -1053,7 +1054,7 @@ describe("container process adapter", () => { const result = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -1096,7 +1097,7 @@ describe("container process adapter", () => { const result = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -1146,7 +1147,7 @@ describe("container process adapter", () => { const result = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -1193,7 +1194,7 @@ describe("container process adapter", () => { const result = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -1249,7 +1250,7 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -1293,7 +1294,7 @@ describe("container process adapter", () => { const result = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -1336,7 +1337,7 @@ describe("container process adapter", () => { const result = yield* Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }); yield* runtime.prepare(image); @@ -1368,7 +1369,7 @@ describe("container process adapter", () => { Effect.gen(function* () { const engine = yield* makeHangingCreateSpawner(); const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }).pipe(Effect.provide(engine.layer)); const launch = yield* Effect.scoped( @@ -1393,7 +1394,7 @@ describe("container process adapter", () => { Effect.gen(function* () { const engine = yield* makeHangingCreateSpawner(); const runtime = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root: ".", }).pipe(Effect.provide(engine.layer)); const launch = yield* Effect.scoped( @@ -1442,7 +1443,7 @@ describe("container process adapter", () => { return Effect.succeed(handle(0, `daemon-at-${command.args[1]}`)); return Effect.die(`Unexpected docker command: ${command.args.join(" ")}`); }); - const target = yield* resolveEngineTarget(spawner); + const target = yield* resolveEngineTarget(spawner, "docker"); // Simulates `docker context use beta` run elsewhere, after this target was resolved. activeContext = "beta"; expect(contextCalls).toHaveLength(1); @@ -1505,7 +1506,7 @@ describe("container process adapter", () => { () => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const target = yield* resolveEngineTarget(spawner); + const target = yield* resolveEngineTarget(spawner, testEngine); const attempts = yield* Ref.make(0); const thirdAttemptObserved = yield* Deferred.make(); // Simulates a registry rejecting a pull while the daemon itself answers normally: this @@ -1594,7 +1595,7 @@ describe("container process adapter", () => { }); // Resolved once, as an owner's own startup would. - const target = yield* resolveEngineTarget(spawner); + const target = yield* resolveEngineTarget(spawner, "docker"); expect(target.argv).toEqual(["--context", "alpha"]); // Simulates `docker context use beta` run elsewhere, after this target was resolved. activeContext = "beta"; @@ -1688,7 +1689,7 @@ const repoDigest = (spawner: ChildProcessSpawnerService["Service"], image: strin Effect.gen(function* () { const child = yield* spawner.spawn( ChildProcess.make( - "docker", + testEngine, ["image", "inspect", "--format", "{{range .RepoDigests}}{{println .}}{{end}}", image], { stdin: "ignore" }, ), @@ -1721,7 +1722,7 @@ const makePullFailureSpawner = ( ChildProcessSpawner.make((command) => { if ( ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "pull" ) return Effect.gen(function* () { @@ -1747,7 +1748,7 @@ const engineHostsScript = `console.log("10.88.0.1\\thost.docker.internal")`; const makeHostGatewayRejectingCreateSpawner = (delegate: ChildProcessSpawnerService["Service"]) => ChildProcessSpawner.make((command) => ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "create" && command.args.includes(`${DOCKER_HOST_ALIAS}:host-gateway`) ? delegate.spawn( @@ -1771,7 +1772,7 @@ const makeHostGatewayProbeSpawner = ( ChildProcessSpawner.make((command) => { if ( !ChildProcess.isStandardCommand(command) || - command.command !== "docker" || + command.command !== testEngine || command.args[0] !== "run" || !command.args.includes("/etc/hosts") ) @@ -1794,7 +1795,7 @@ const makeStopFailureSpawner = ( ChildProcessSpawner.make((command) => { if ( ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "stop" ) { return Effect.gen(function* () { @@ -1819,7 +1820,7 @@ const makeLogFollowerSpawner = ( ChildProcessSpawner.make((command) => { if ( ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "logs" ) return Effect.gen(function* () { @@ -1838,7 +1839,7 @@ const makeLostRemoveResultSpawner = ( ChildProcessSpawner.make((command) => { if ( ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "rm" ) { return Effect.gen(function* () { @@ -1872,7 +1873,7 @@ const makeRemoveFailureSpawner = ( if ( failProbe !== undefined && ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "ps" && // Excludes the host-gateway probe's own throwaway "-task-" container, created and awaited // absent during `launch`, before this test's own instance container ever reaches `remove`. @@ -1892,7 +1893,7 @@ const makeRemoveFailureSpawner = ( } if ( ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "rm" ) { return Effect.gen(function* () { @@ -1918,7 +1919,7 @@ const makePendingRemoveSpawner = ( ChildProcessSpawner.make((command) => { if ( ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "ps" && // Excludes the host-gateway probe's own throwaway "-task-" container, created and awaited // absent during `launch`, before this test's own instance container ever reaches `remove`. @@ -1937,7 +1938,7 @@ const makePendingRemoveSpawner = ( } if ( ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "rm" ) { return Effect.gen(function* () { @@ -1970,7 +1971,7 @@ const makeInterruptedRemoveSpawner = ( ChildProcessSpawner.make((command) => { if ( ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "rm" ) { return Effect.gen(function* () { @@ -2006,7 +2007,7 @@ const makeCreateFailureSpawner = ( ChildProcessSpawner.make((command) => { if ( ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "create" ) { return Effect.gen(function* () { @@ -2041,7 +2042,7 @@ const makePendingCreateSpawner = ( present: Ref.Ref, ) => ChildProcessSpawner.make((command) => { - if (!ChildProcess.isStandardCommand(command) || command.command !== "docker") + if (!ChildProcess.isStandardCommand(command) || command.command !== testEngine) return delegate.spawn(command); if (command.args[0] === "stop" || command.args[0] === "rm") { return Effect.gen(function* () { @@ -2142,7 +2143,7 @@ const exists = (id: string) => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", ["inspect", id], { + ChildProcess.make(testEngine, ["inspect", id], { stdin: "ignore", stdout: "ignore", stderr: "ignore", @@ -2155,7 +2156,7 @@ const inspectLabels = (id: string) => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", ["inspect", "--format={{json .Config.Labels}}", id], { + ChildProcess.make(testEngine, ["inspect", "--format={{json .Config.Labels}}", id], { stdin: "ignore", }), ); @@ -2169,7 +2170,7 @@ const inspectExtraHosts = (id: string) => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", ["inspect", "--format={{json .HostConfig.ExtraHosts}}", id], { + ChildProcess.make(testEngine, ["inspect", "--format={{json .HostConfig.ExtraHosts}}", id], { stdin: "ignore", }), ); @@ -2183,7 +2184,7 @@ const removeExternally = (id: string) => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", ["rm", "--force", id], { + ChildProcess.make(testEngine, ["rm", "--force", id], { stdin: "ignore", stdout: "ignore", stderr: "ignore", @@ -2192,7 +2193,7 @@ const removeExternally = (id: string) => const code = yield* child.exitCode; if (Number(code) !== 0) return yield* new ContainerTestError({ - message: `docker rm --force exited with ${String(code)}`, + message: `${testEngine} rm --force exited with ${String(code)}`, }); }); @@ -2201,7 +2202,7 @@ const idsByInstance = (instanceId: string) => const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( ChildProcess.make( - "docker", + testEngine, [ "ps", "--all", @@ -2226,7 +2227,7 @@ const removeByInstance = (instanceId: string) => if (ids.length === 0) return; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", ["rm", "--force", ...ids], { stdin: "ignore" }), + ChildProcess.make(testEngine, ["rm", "--force", ...ids], { stdin: "ignore" }), ); const [stderr, code] = yield* Effect.all([ child.stderr.pipe(Stream.decodeText, Stream.mkString), @@ -2243,7 +2244,7 @@ const running = (id: string) => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", ["inspect", "--format={{.State.Running}}", id], { + ChildProcess.make(testEngine, ["inspect", "--format={{.State.Running}}", id], { stdin: "ignore", }), ); diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index 582992dc52..c85fc0556d 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -260,7 +260,7 @@ const resolvePodmanTarget = Effect.fn("Container.resolvePodmanTarget")(function* */ export const resolveEngineTarget = Effect.fn("Container.resolveEngineTarget")(function* ( spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], - engine: ContainerEngine = "docker", + engine: ContainerEngine, ): Effect.fn.Return { yield* Effect.annotateCurrentSpan("container.engine", engine); if (engine === "podman") return yield* resolvePodmanTarget(spawner); diff --git a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts index 18b3a92d1a..c55bddae8f 100644 --- a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts +++ b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts @@ -7,7 +7,7 @@ import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; -import { dockerEngineTarget } from "../../tests/engine-target.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; const options = (root: string) => ({ stackId: "catalog-analytics", @@ -19,8 +19,8 @@ const options = (root: string) => ({ const dockerOptions = (root: string) => ({ ...options(root), - runtime: "docker" as const, - engineTarget: dockerEngineTarget, + runtime: testEngine, + engineTarget, }); describe("service catalog", () => { diff --git a/packages/stack/src/services/AuthStorage.integration.test.ts b/packages/stack/src/services/AuthStorage.integration.test.ts index 871efe661c..d271baabd3 100644 --- a/packages/stack/src/services/AuthStorage.integration.test.ts +++ b/packages/stack/src/services/AuthStorage.integration.test.ts @@ -8,7 +8,7 @@ import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; -import { dockerEngineTarget } from "../../tests/engine-target.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; const options = (root: string) => ({ stackId: "catalog-auth-storage", @@ -20,8 +20,8 @@ const options = (root: string) => ({ const dockerOptions = (root: string) => ({ ...options(root), - runtime: "docker" as const, - engineTarget: dockerEngineTarget, + runtime: testEngine, + engineTarget, }); describe("service catalog", () => { diff --git a/packages/stack/src/services/Database.integration.test.ts b/packages/stack/src/services/Database.integration.test.ts index 31bfdc7f5e..bcacc1240a 100644 --- a/packages/stack/src/services/Database.integration.test.ts +++ b/packages/stack/src/services/Database.integration.test.ts @@ -14,14 +14,14 @@ import { Ref, Stream, } from "effect"; -import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { createHash } from "node:crypto"; import { tmpdir } from "node:os"; import { DEFAULT_POSTGRES_ROOT_KEY } from "../Defaults.ts"; import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeDatabase, type BackendEndpoint, type DatabaseConfig } from "./Database.ts"; -import { makeDockerDatabaseRoot, runDocker } from "../../tests/docker-fixture.ts"; -import { dockerEngineTarget } from "../../tests/engine-target.ts"; +import { makeDockerDatabaseRoot, runEngine } from "../../tests/docker-fixture.ts"; +import { observeContainerStop, cleanStopEvents } from "../../tests/engine-events.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; const config: DatabaseConfig = { version: "17", @@ -59,8 +59,8 @@ const query = ( describe("database component", { timeout: 180_000 }, () => { for (const target of [ { runtime: "native", version: "17" }, - { runtime: "docker", version: "15" }, - { runtime: "docker", version: "17" }, + { runtime: testEngine, version: "15" }, + { runtime: testEngine, version: "17" }, ] as const) it.live( `preserves default encryption keys and Vault secrets after ${target.runtime} PostgreSQL ${target.version} recreation`, @@ -70,7 +70,7 @@ describe("database component", { timeout: 180_000 }, () => { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const root = - target.runtime === "docker" + target.runtime !== "native" ? yield* makeDockerDatabaseRoot("stack-default-key-", "default-key-test") : yield* fs.makeTempDirectoryScoped({ prefix: "stack-default-key-" }); const recipe = yield* makeDatabase({ @@ -79,7 +79,7 @@ describe("database component", { timeout: 180_000 }, () => { root, cacheRoot: artifactCacheRoot, runtime: target.runtime, - ...(target.runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), + ...(target.runtime !== "native" ? { engineTarget } : {}), }); const defaults: DatabaseConfig = { healthTimeoutMs: 120_000, @@ -433,8 +433,8 @@ describe("database component", { timeout: 180_000 }, () => { instanceId: "database", root, cacheRoot, - runtime: "docker", - engineTarget: dockerEngineTarget, + runtime: testEngine, + engineTarget, }); const service = yield* makeStandaloneService(database.definition, { id: "database:container", @@ -489,8 +489,8 @@ describe("database component", { timeout: 180_000 }, () => { instanceId: "database", root, cacheRoot, - runtime: "docker", - engineTarget: dockerEngineTarget, + runtime: testEngine, + engineTarget, }); const replacementPassword = Redacted.make("reopened-target-password"); const reopenedService = yield* makeStandaloneService(reopened.definition, { @@ -514,49 +514,53 @@ describe("database component", { timeout: 180_000 }, () => { ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); - it.live("groups the database and its storage helper under one compose project", () => - Effect.scoped( - Effect.gen(function* () { - const path = yield* Path.Path; - const stackId = "stack-compose-group"; - const root = yield* makeDockerDatabaseRoot("stack-database-group-", stackId); - const database = yield* makeDatabase({ - stackId, - instanceId: "database", - project: "my.app", - root, - cacheRoot: artifactCacheRoot, - runtime: "docker", - engineTarget: dockerEngineTarget, - }); - const service = yield* makeStandaloneService(database.definition, { - id: "database:group", - config, - }); - yield* service.start; - yield* service.ready; - const listed = yield* runDocker([ - "ps", - "--filter", - `label=com.supabase.stack-root=${path.resolve(root)}`, - "--format", - '{{.Names}}|{{.Label "com.docker.compose.project"}}|{{.Label "com.docker.compose.service"}}', - ]); - const rows = listed.output - .split("\n") - .filter((line) => line.trim().length > 0) - .map((line) => line.trim().split("|")); - expect(rows.some(([name]) => name?.startsWith("supabase-db-helper-"))).toBe(true); - expect(new Set(rows.map(([, project]) => project))).toEqual( - new Set(["supabase-my-app-stack-compos"]), - ); - expect(new Set(rows.map(([, , group]) => group))).toEqual( - new Set(["database", "database-helper"]), - ); - yield* service.destroy; - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), - ); + // The storage helper joins the compose project only with the Docker volume backend. + describe.runIf(testEngine === "docker")("Docker volume storage", () => { + it.live("groups the database and its storage helper under one compose project", () => + Effect.scoped( + Effect.gen(function* () { + const path = yield* Path.Path; + const stackId = "stack-compose-group"; + const root = yield* makeDockerDatabaseRoot("stack-database-group-", stackId); + const database = yield* makeDatabase({ + stackId, + instanceId: "database", + project: "my.app", + root, + cacheRoot: artifactCacheRoot, + runtime: testEngine, + engineTarget, + }); + const service = yield* makeStandaloneService(database.definition, { + id: "database:group", + config, + }); + yield* service.start; + yield* service.ready; + const listed = yield* runEngine([ + ...engineTarget.argv, + "ps", + "--filter", + `label=com.supabase.stack-root=${path.resolve(root)}`, + "--format", + '{{.Names}}|{{.Label "com.docker.compose.project"}}|{{.Label "com.docker.compose.service"}}', + ]); + const rows = listed.output + .split("\n") + .filter((line) => line.trim().length > 0) + .map((line) => line.trim().split("|")); + expect(rows.some(([name]) => name?.startsWith("supabase-db-helper-"))).toBe(true); + expect(new Set(rows.map(([, project]) => project))).toEqual( + new Set(["supabase-my-app-stack-compos"]), + ); + expect(new Set(rows.map(([, , group]) => group))).toEqual( + new Set(["database", "database-helper"]), + ); + yield* service.destroy; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); + }); it.live("shuts PostgreSQL down fast while a client stays connected across stop", () => Effect.scoped( @@ -569,8 +573,8 @@ describe("database component", { timeout: 180_000 }, () => { instanceId: "database", root, cacheRoot: artifactCacheRoot, - runtime: "docker", - engineTarget: dockerEngineTarget, + runtime: testEngine, + engineTarget, }); const service = yield* makeStandaloneService(database.definition, { id: "database:shutdown", @@ -580,7 +584,8 @@ describe("database component", { timeout: 180_000 }, () => { yield* service.ready; const endpoint = yield* database.endpoint; if (endpoint.kind !== "tcp") return yield* Effect.die("Expected a TCP endpoint"); - const listed = yield* runDocker([ + const listed = yield* runEngine([ + ...engineTarget.argv, "ps", "--filter", `label=com.supabase.stack-root=${path.resolve(root)}`, @@ -596,42 +601,15 @@ describe("database component", { timeout: 180_000 }, () => { .filter((name) => name.length > 0 && !name.startsWith("supabase-db-helper-")); expect(containers).toHaveLength(1); const container = containers.join(""); - const startedAt = yield* runDocker([ + const startedAt = yield* runEngine([ + ...engineTarget.argv, "inspect", "--format", "{{.State.StartedAt}}", container, ]); - // A bounded `docker events --until` replay intermittently returns nothing on Docker Desktop, - // so subscribe live before stop; `--since` replays anything logged before the stream attaches. - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const events = yield* spawner.spawn( - ChildProcess.make( - "docker", - [ - "events", - "--since", - startedAt.output.trim(), - "--filter", - `container=${container}`, - "--filter", - "event=kill", - "--filter", - "event=die", - "--format", - '{{.Action}} {{index .Actor.Attributes "signal"}}{{index .Actor.Attributes "exitCode"}}', - ], - { stdin: "ignore", stdout: "pipe", stderr: "ignore" }, - ), - ); - const observed = yield* events.stdout.pipe( - Stream.decodeText, - Stream.splitLines, - Stream.takeUntil((line) => line.startsWith("die ")), - Stream.runCollect, - Effect.forkScoped, - ); + const observed = yield* observeContainerStop(container, startedAt.output.trim()); yield* Effect.scoped( Effect.gen(function* () { @@ -655,7 +633,7 @@ describe("database component", { timeout: 180_000 }, () => { expect( yield* Fiber.join(observed), "stop sends SIGINT and PostgreSQL exits cleanly", - ).toEqual(["kill 2", "die 0"]); + ).toEqual(cleanStopEvents); yield* service.destroy; }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), diff --git a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts index 3d3a610613..add24aa49f 100644 --- a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts +++ b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts @@ -8,6 +8,7 @@ import { makeDockerDatabaseStorage } from "../storage/DockerDatabaseStorage.ts"; import { makeDockerHelperRegistry } from "../storage/DockerHelperRegistry.ts"; import { shellQuote } from "../storage/DockerSnapshotBackend.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; +import { testEngine } from "../../tests/test-engine.ts"; import { makeDatabaseSnapshots, type SnapshotScope } from "./DatabaseSnapshot.ts"; // Derived from the real catalog (not hardcoded), so a Postgres pin bump never makes this go stale. @@ -183,7 +184,7 @@ const docker = Effect.fnUntraced(function* () { const root = yield* makeDockerDatabaseRoot("database-snapshot-contract-", stackId); const cacheRoot = path.resolve(root, "../../../cache"); const image = (yield* resolveArtifact({ service: "database", version }).pipe(Effect.orDie)).image; - const target = yield* resolveEngineTarget(spawner); + const target = yield* resolveEngineTarget(spawner, "docker"); const container = yield* makeContainerRuntime({ target, root, @@ -299,9 +300,10 @@ const docker = Effect.fnUntraced(function* () { return engine; }); +// The Docker case exercises the Docker volume backend; Podman keeps data in host directories. const engines = [ { name: "native", make: native }, - { name: "docker", make: docker }, + ...(testEngine === "docker" ? [{ name: "docker", make: docker }] : []), ]; const Descriptor = Schema.fromJsonString(Schema.Struct({ logicalKey: Schema.String })); diff --git a/packages/stack/src/services/Functions.integration.test.ts b/packages/stack/src/services/Functions.integration.test.ts index 7ed37b2529..a91b59c1a2 100644 --- a/packages/stack/src/services/Functions.integration.test.ts +++ b/packages/stack/src/services/Functions.integration.test.ts @@ -22,7 +22,7 @@ import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import * as Functions from "./Functions.ts"; import { makeProcessRecipe } from "./ProcessRecipe.ts"; -import { dockerEngineTarget } from "../../tests/engine-target.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; const options = (root: string) => ({ @@ -35,19 +35,19 @@ const options = (root: string) => ({ const dockerOptions = (root: string) => ({ ...options(root), - runtime: "docker" as const, - engineTarget: dockerEngineTarget, + runtime: testEngine, + engineTarget, }); -const dockerInfo = Effect.scoped( +const engineInfo = Effect.scoped( Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", ["info"], { stdout: "pipe", stderr: "pipe" }), + ChildProcess.make(testEngine, ["info"], { stdout: "pipe", stderr: "pipe" }), ); return yield* Stream.mkString(Stream.decodeText(child.all)); }), -).pipe(Effect.orElseSucceed(() => "docker info unavailable")); +).pipe(Effect.orElseSucceed(() => `${testEngine} info unavailable`)); describe("service catalog", () => { it.live( @@ -451,7 +451,7 @@ describe("service catalog", () => { ); }); -for (const runtime of ["native", "docker"] as const) { +for (const runtime of ["native", testEngine] as const) { it.live( `serves configured function files, secrets and JWT policies in ${runtime}`, () => @@ -521,7 +521,7 @@ for (const runtime of ["native", "docker"] as const) { stackId: "c".repeat(64), instanceId: "configured", runtime, - ...(runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), + ...(runtime !== "native" ? { engineTarget } : {}), cacheRoot: "/tmp/supabase-stack-artifacts", }, ); @@ -575,8 +575,8 @@ for (const runtime of ["native", "docker"] as const) { Effect.tapCause(() => Effect.gen(function* () { yield* Effect.logError(`Functions ${runtime} logs:\n${yield* Ref.get(logs)}`); - if (runtime === "docker") - yield* Effect.logError(`docker info:\n${yield* dockerInfo}`); + if (runtime !== "native") + yield* Effect.logError(`${testEngine} info:\n${yield* engineInfo}`); }), ), ); diff --git a/packages/stack/src/services/Mail.integration.test.ts b/packages/stack/src/services/Mail.integration.test.ts index 6f320190a0..4965058a7a 100644 --- a/packages/stack/src/services/Mail.integration.test.ts +++ b/packages/stack/src/services/Mail.integration.test.ts @@ -4,7 +4,7 @@ import { Effect, Exit, FileSystem, Layer, Path } from "effect"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; -import { dockerEngineTarget } from "../../tests/engine-target.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; const options = (root: string) => ({ @@ -17,8 +17,8 @@ const options = (root: string) => ({ const dockerOptions = (root: string) => ({ ...options(root), - runtime: "docker" as const, - engineTarget: dockerEngineTarget, + runtime: testEngine, + engineTarget, }); interface MailpitMessages { diff --git a/packages/stack/src/services/Pooler.integration.test.ts b/packages/stack/src/services/Pooler.integration.test.ts index b155189bcb..f35a295426 100644 --- a/packages/stack/src/services/Pooler.integration.test.ts +++ b/packages/stack/src/services/Pooler.integration.test.ts @@ -7,7 +7,7 @@ import { ProxyError } from "../Proxy.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { dockerEngineTarget } from "../../tests/engine-target.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; const options = (root: string) => ({ stackId: "catalog-pooler", @@ -20,8 +20,8 @@ const options = (root: string) => ({ const dockerOptions = (root: string) => ({ ...options(root), cacheRoot: `${root}/cache`, - runtime: "docker" as const, - engineTarget: dockerEngineTarget, + runtime: testEngine, + engineTarget, }); describe("service catalog", () => { @@ -56,7 +56,7 @@ describe("service catalog", () => { const databaseRelay = yield* makeDockerTcpRelay(databaseRecipe.endpoint("sql")); const dockerDatabaseUrl = `postgresql://supabase_admin:postgres@${databaseRelay.host}:${databaseRelay.port}/_supabase`; const nativeDatabaseUrl = `postgresql://supabase_admin:postgres@${databaseEndpoint.host}:${databaseEndpoint.port}/_supabase`; - for (const runtime of ["native", "docker"] as const) { + for (const runtime of ["native", testEngine] as const) { for (const poolMode of ["transaction", "session"] as const) { const tenant = `catalog-${runtime}-${poolMode}`; const poolerRecipe = yield* makeServiceRecipe( diff --git a/packages/stack/src/services/Realtime.integration.test.ts b/packages/stack/src/services/Realtime.integration.test.ts index ae9d5f9db6..1daa650085 100644 --- a/packages/stack/src/services/Realtime.integration.test.ts +++ b/packages/stack/src/services/Realtime.integration.test.ts @@ -6,7 +6,7 @@ import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { dockerEngineTarget } from "../../tests/engine-target.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; const options = (root: string) => ({ @@ -19,8 +19,8 @@ const options = (root: string) => ({ const dockerOptions = (root: string) => ({ ...options(root), - runtime: "docker" as const, - engineTarget: dockerEngineTarget, + runtime: testEngine, + engineTarget, }); describe("service catalog", () => { diff --git a/packages/stack/src/services/Rest.integration.test.ts b/packages/stack/src/services/Rest.integration.test.ts index b465d68cac..55e8a56867 100644 --- a/packages/stack/src/services/Rest.integration.test.ts +++ b/packages/stack/src/services/Rest.integration.test.ts @@ -11,7 +11,7 @@ import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { ProxyError } from "../Proxy.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; -import { dockerEngineTarget } from "../../tests/engine-target.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; const makeTestState = (root: string) => @@ -40,8 +40,8 @@ const options = (root: string) => ({ const dockerOptions = (root: string) => ({ ...options(root), - runtime: "docker" as const, - engineTarget: dockerEngineTarget, + runtime: testEngine, + engineTarget, }); describe("service catalog", () => { @@ -58,12 +58,12 @@ describe("service catalog", () => { yield* state.save({ id: stackId, identity: { projectRoot: root, branchContext: "test", stackName: "catalog" }, - runtime: "docker", + runtime: testEngine, instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, }); - const network = yield* makeTestNetwork({ stackId, runtime: "docker", state }); + const network = yield* makeTestNetwork({ stackId, runtime: testEngine, state }); const secret = "catalog-rest-secret-with-at-least-32-chars"; const databaseRecipe = yield* makeServiceRecipe( { diff --git a/packages/stack/src/services/StorageGateway.integration.test.ts b/packages/stack/src/services/StorageGateway.integration.test.ts index 1e60902fc1..e41a0af9d2 100644 --- a/packages/stack/src/services/StorageGateway.integration.test.ts +++ b/packages/stack/src/services/StorageGateway.integration.test.ts @@ -12,7 +12,7 @@ import * as StackNamespace from "../StackNamespace.ts"; import type { SavedStack } from "../StackNamespace.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { ownerFor } from "../../tests/owner-rpc.ts"; -import { dockerEngineTarget } from "../../tests/engine-target.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; const jwtSecret = "storage-gateway-secret-with-at-least-32-chars"; @@ -69,7 +69,7 @@ const serveStorage = Effect.fnUntraced(function* (runtime: SavedStack["runtime"] state, root: dataRoot, cacheRoot, - ...(runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), + ...(runtime !== "native" ? { engineTarget } : {}), }); yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); const created = yield* owner.rpc.supabaseComposition({ @@ -100,7 +100,7 @@ const serveStorage = Effect.fnUntraced(function* (runtime: SavedStack["runtime"] return { url, serviceRoleKey, storageRoot }; }); -for (const runtime of ["native", "docker"] as const) +for (const runtime of ["native", testEngine] as const) it.live( `accepts S3 requests signed over the gateway path and resumes TUS uploads there (${runtime})`, () => @@ -162,7 +162,7 @@ for (const runtime of ["native", "docker"] as const) expect(patch.status, yield* patch.text).toBe(204); expect(patch.headers["upload-offset"]).toBe("5"); - if (runtime === "docker") { + if (runtime !== "native") { // The container writes into storageRoot, a borrowed host directory; the host user // must still own what it created there, not the engine's own root (see Container's // `user` field). diff --git a/packages/stack/src/services/Vector.integration.test.ts b/packages/stack/src/services/Vector.integration.test.ts index aaddd903fd..3fca719cd7 100644 --- a/packages/stack/src/services/Vector.integration.test.ts +++ b/packages/stack/src/services/Vector.integration.test.ts @@ -5,20 +5,21 @@ import { Effect, FileSystem, Layer } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; -import { dockerEngineTarget } from "../../tests/engine-target.ts"; +import type { ContainerEngine } from "../runtime/Container.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; -const options = (root: string, runtime: "docker" | "native") => ({ +const options = (root: string, runtime: ContainerEngine | "native") => ({ stackId: "catalog-test", instanceId: "vector", root, cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, runtime, - ...(runtime === "docker" ? { engineTarget: dockerEngineTarget } : {}), + ...(runtime !== "native" ? { engineTarget } : {}), }); describe("vector recipe", () => { - for (const runtime of ["docker", "native"] as const) { + for (const runtime of [testEngine, "native"] as const) { it.live( `serves health without a custom config and cleans up on destroy, including interrupted writes (${runtime})`, () => @@ -80,7 +81,7 @@ describe("vector recipe", () => { config: { analyticsUrl: "http://analytics", configPath: pipeline }, endpoints: { http: { port: "auto" } }, }, - options(root, "docker"), + options(root, testEngine), ); const vector = yield* makeStandaloneService(recipe.definition, { id: "vector", @@ -110,7 +111,7 @@ describe("vector recipe", () => { config: { analyticsUrl: "http://analytics" }, endpoints: { http: { port: "auto" } }, }, - options(root, "docker"), + options(root, testEngine), ); const vector = yield* makeStandaloneService(recipe.definition, { id: "vector", @@ -164,7 +165,7 @@ describe("vector recipe", () => { config: { analyticsUrl: "http://analytics", configPath }, endpoints: { http: { port: "auto" } }, }, - options(root, "docker"), + options(root, testEngine), ); const vector = yield* makeStandaloneService(recipe.definition, { id: "vector", diff --git a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts index f44c968ec9..26cdd7817a 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts @@ -25,11 +25,13 @@ import { makeContainerRuntime, type EngineTarget } from "../runtime/Container.ts import { makeDatabaseSnapshots } from "../services/DatabaseSnapshot.ts"; import { makeDockerDatabaseStorage } from "./DockerDatabaseStorage.ts"; import { makeDockerHelperRegistry } from "./DockerHelperRegistry.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; import { removeTestRunVolumes } from "../../tests/docker-volume-run.ts"; import type { DockerHelperRegistry } from "./DockerHelperRegistry.ts"; // An unpinned target: these tests exercise storage marker/volume logic, not endpoint pinning. const dockerTarget: EngineTarget = { engine: "docker", argv: [], daemonId: "test-daemon-id" }; +const containerTarget: EngineTarget = { ...engineTarget, argv: [] }; const postgresImage = (version: string) => resolveArtifact({ service: "database", version: postgresVersion(version) }).pipe( Effect.map(({ image }) => image), @@ -110,11 +112,11 @@ const fakeHelperEngine = ( return { commands, layer: Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner) }; }; -const docker = Effect.fn("DockerStorageTest.docker")((args: ReadonlyArray) => +const engine = Effect.fn("StorageTest.engine")((args: ReadonlyArray) => Effect.scoped( Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const child = yield* spawner.spawn(ChildProcess.make("docker", args, { stdin: "ignore" })); + const child = yield* spawner.spawn(ChildProcess.make(testEngine, args, { stdin: "ignore" })); const [stdout, stderr, code] = yield* Effect.all( [ child.stdout.pipe( @@ -137,7 +139,7 @@ const docker = Effect.fn("DockerStorageTest.docker")((args: ReadonlyArray `'${value.replaceAll("'", "'\\''")}'`; -describe("Docker database storage", { timeout: 120_000 }, () => { - it.live("waits for helper creation to settle before cleaning up an interrupted operation", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-helper-create-cancel-" }); - const started = yield* Deferred.make(); - const release = yield* Deferred.make(); - const present = yield* Ref.make(false); - const removals = yield* Ref.make(0); - const spawner = ChildProcessSpawner.make((command) => - Effect.gen(function* () { - if (!ChildProcess.isStandardCommand(command)) - return yield* Effect.die("Unexpected command"); - const creating = command.args[0] === "run"; - const identity = command.args[0] === "info"; - if (creating) yield* Deferred.succeed(started, undefined); - if (command.args[0] === "rm") { - yield* Ref.update(removals, (count) => count + 1); - yield* Ref.set(present, false); - } - return ChildProcessSpawner.makeHandle({ - pid: ChildProcessSpawner.ProcessId(0), - exitCode: (creating - ? Deferred.await(release).pipe(Effect.andThen(Ref.set(present, true))) - : Effect.void - ).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), - isRunning: Effect.succeed(false), - kill: () => Effect.void, - stdin: Sink.drain, - stdout: creating - ? Stream.succeed(new TextEncoder().encode("a".repeat(64))) - : identity - ? Stream.succeed(new TextEncoder().encode('{"host":{"id":"fake-daemon-id"}}')) - : Stream.empty, - stderr: Stream.empty, - all: Stream.empty, - getInputFd: () => Sink.drain, - getOutputFd: () => Stream.empty, - unref: Effect.succeed(Effect.void), - }); - }), - ); - const storage = yield* makeDockerDatabaseStorage({ - runtime: "docker", - target: dockerTarget, - stackId: "helper-create-cancel", - instanceId: "database", - instanceRoot: root, - root, - cacheRoot: path.join(root, "cache"), - fs, - path, - crypto, - spawner, - container: { - prepare: () => Effect.void, - prepareImage: (image) => Effect.succeed(image), - launch: () => Effect.die("unused"), - launchCommand: () => Effect.die("unused"), - }, - }); - yield* storage.prepare("17"); - const operation = yield* storage.removeData("17").pipe(Effect.forkScoped); - yield* Deferred.await(started); - yield* Effect.sync(() => operation.interruptUnsafe()); - yield* Effect.yieldNow; - yield* Deferred.succeed(release, undefined); - yield* Fiber.await(operation); - expect(yield* Ref.get(removals)).toBe(1); - expect(yield* Ref.get(present)).toBe(false); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("starts a storage helper with the mirror image selected during preparation", () => { - const engine = fakeHelperEngine(); - return Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "storage-helper-mirror-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const instanceRoot = path.join(storageRoot, "database"); - yield* fs.makeDirectory(instanceRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ - target: dockerTarget, - root, - imageMirrors: () => [helperMirror], - }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const storage = yield* makeDockerDatabaseStorage({ - runtime: "docker", - target: dockerTarget, - stackId: "storage-helper-mirror", - instanceId: "database", - instanceRoot, - root: storageRoot, - cacheRoot, - fs, - path, - crypto, - container, - spawner, - }); - - yield* storage.prepare("17"); - yield* storage.removeData("17"); - - const run = engine.commands.find((args) => args[0] === "run"); - const shellIndex = run?.indexOf("/bin/sh") ?? -1; - expect(run?.[shellIndex - 1]).toBe(helperMirror); - expect( - engine.commands.filter((args) => args[0] === "pull").map((args) => args.at(-1)), - ).toEqual([expect.stringContaining("ghcr.io/supabase/cli/postgres:"), helperMirror]); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); - }); - - it.live("starts a shared volume helper with the mirror image selected during preparation", () => { - const engine = fakeHelperEngine(undefined, { volumePresent: true }); - return Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "storage-shared-helper-mirror-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const instanceRoot = path.join(storageRoot, "database"); - yield* fs.makeDirectory(instanceRoot, { recursive: true }); - yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const stackId = "shared-helper-mirror"; - const instanceId = "database"; - const cachePath = yield* fs.realPath(cacheRoot); - const cacheHash = yield* crypto.digest("SHA-256", new TextEncoder().encode(cachePath)); - const cacheNamespace = `cache-${Array.from(cacheHash, (byte) => - byte.toString(16).padStart(2, "0"), - ) - .join("") - .slice(0, 32)}`; - // Matches the volume name the engine target's own identity resolves, the same way - // DockerDatabaseStorage derives it: a stable digest of the state root and daemon id. - const statePath = yield* fs.realPath(path.join(root, "state")); - const stateHash = yield* crypto.digest( - "SHA-256", - new TextEncoder().encode(`${statePath}\0${dockerTarget.daemonId}`), - ); - const stateDigest = Array.from(stateHash, (byte) => byte.toString(16).padStart(2, "0")) - .join("") - .slice(0, 32); - const marker = yield* Schema.encodeEffect(Schema.fromJsonString(Marker))({ - backend: "docker", - volume: `supabase-db-${stateDigest}`, - namespace: `instance-${stackId}-${instanceId}`, - cacheNamespace, - daemonId: dockerTarget.daemonId, - initialized: false, - }); - yield* fs.writeFileString( - path.join(instanceRoot, ".supabase-database-storage.json"), - marker, - ); - const container = yield* makeContainerRuntime({ - target: dockerTarget, - root, - imageMirrors: () => [helperMirror], - }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const helpers = yield* makeDockerHelperRegistry("mirror-test"); - const storage = yield* makeDockerDatabaseStorage({ - runtime: "docker", - target: dockerTarget, - stackId, - instanceId, - instanceRoot, - root: storageRoot, - cacheRoot, - fs, - path, - crypto, - container, - spawner, - helpers, - }); - - yield* storage.prepare("17"); - - const run = engine.commands.find((args) => args[0] === "run"); - const shellIndex = run?.indexOf("/bin/sh") ?? -1; - expect(run?.[shellIndex - 1]).toBe(helperMirror); - expect( - engine.commands.filter((args) => args[0] === "pull").map((args) => args.at(-1)), - ).toEqual([expect.stringContaining("ghcr.io/supabase/cli/postgres:"), helperMirror]); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); - }); - - it.live("round-trips PostgreSQL 15 data with its catalog image", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const helperImage = yield* postgresImage("15"); - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-pg15-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const sourceRoot = path.join(storageRoot, "source"); - const targetRoot = path.join(storageRoot, "target"); - yield* fs.makeDirectory(sourceRoot, { recursive: true }); - yield* fs.makeDirectory(targetRoot, { recursive: true }); - yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ - target: dockerTarget, - root, - }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const stackId = `storage-pg15-${yield* crypto.randomUUIDv4}`; - const makeStorage = (instanceId: string, instanceRoot: string) => - makeDockerDatabaseStorage({ +// The volume backend, helper registry and volume-labelling contracts exist only on Docker; Podman +// always stores data in host directories, covered by the host storage suite below. +describe.runIf(testEngine === "docker")( + "Docker volume database storage", + { timeout: 120_000 }, + () => { + it.live("waits for helper creation to settle before cleaning up an interrupted operation", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "docker-helper-create-cancel-", + }); + const started = yield* Deferred.make(); + const release = yield* Deferred.make(); + const present = yield* Ref.make(false); + const removals = yield* Ref.make(0); + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if (!ChildProcess.isStandardCommand(command)) + return yield* Effect.die("Unexpected command"); + const creating = command.args[0] === "run"; + const identity = command.args[0] === "info"; + if (creating) yield* Deferred.succeed(started, undefined); + if (command.args[0] === "rm") { + yield* Ref.update(removals, (count) => count + 1); + yield* Ref.set(present, false); + } + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(0), + exitCode: (creating + ? Deferred.await(release).pipe(Effect.andThen(Ref.set(present, true))) + : Effect.void + ).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + stdout: creating + ? Stream.succeed(new TextEncoder().encode("a".repeat(64))) + : identity + ? Stream.succeed(new TextEncoder().encode('{"host":{"id":"fake-daemon-id"}}')) + : Stream.empty, + stderr: Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + }), + ); + const storage = yield* makeDockerDatabaseStorage({ runtime: "docker", target: dockerTarget, - stackId, - instanceId, - instanceRoot, - root: storageRoot, - cacheRoot, + stackId: "helper-create-cancel", + instanceId: "database", + instanceRoot: root, + root, + cacheRoot: path.join(root, "cache"), fs, path, crypto, - container, spawner, + container: { + prepare: () => Effect.void, + prepareImage: (image) => Effect.succeed(image), + launch: () => Effect.die("unused"), + launchCommand: () => Effect.die("unused"), + }, }); - const source = yield* makeStorage("source", sourceRoot); - const target = yield* makeStorage("target", targetRoot); - const ownerScope = yield* Scope.Scope; - yield* Scope.addFinalizer( - ownerScope, - Effect.gen(function* () { - yield* source.destroyData("15").pipe(Effect.ignore); - yield* target.destroyData("15").pipe(Effect.ignore); - const marker = yield* fs - .readFileString(path.join(sourceRoot, ".supabase-database-storage.json")) - .pipe(Effect.option); - if (Option.isSome(marker)) { - const parsed = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - marker.value, - ).pipe(Effect.option); - if (Option.isSome(parsed) && parsed.value.volume !== undefined) - yield* docker(["volume", "rm", parsed.value.volume]).pipe(Effect.ignore); - } - }).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - Effect.ignore, - ), - ); - - yield* source.prepare("15"); - const sourceMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), - ); - if (sourceMarker.backend !== "docker" || sourceMarker.volume === undefined) - return yield* new DockerTestError({ message: "Docker test selected host fallback" }); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceMarker.volume},dst=/store`, - helperImage, - "/bin/sh", - "-c", - `set -eu; printf 15 > ${quote(`/store/${sourceMarker.namespace}/data/PG_VERSION`)}; printf pg15-source > ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}`, - ]); - yield* source.markInitialized("15"); - yield* fs.writeFileString( - path.join(sourceRoot, ".supabase-database-ready.json"), - '{"version":"15","runtime":"docker","profile":"supabase"}', - ); - yield* source.saveSnapshot("15", "restore-me"); - - yield* target.prepare("15"); - expect(yield* target.restoreSnapshot("15", "restore-me")).toBe(true); - const targetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(path.join(targetRoot, ".supabase-database-storage.json")), - ); - expect(targetMarker.volume).toBe(sourceMarker.volume); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceMarker.volume},dst=/store,volume-subpath=${targetMarker.namespace}/data`, - helperImage, - "/bin/sh", - "-c", - 'test "$(cat /store/PG_VERSION)" = 15 && test "$(cat /store/fixture)" = pg15-source', - ]); - yield* target.destroyData("15"); - yield* source.destroyData("15"); - yield* docker(["volume", "rm", sourceMarker.volume]); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); + yield* storage.prepare("17"); + const operation = yield* storage.removeData("17").pipe(Effect.forkScoped); + yield* Deferred.await(started); + yield* Effect.sync(() => operation.interruptUnsafe()); + yield* Effect.yieldNow; + yield* Deferred.succeed(release, undefined); + yield* Fiber.await(operation); + expect(yield* Ref.get(removals)).toBe(1); + expect(yield* Ref.get(present)).toBe(false); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); - it.live("round-trips stopped data and protects managed namespaces", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const helperImage = yield* postgresImage("17"); - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-" }); - const stateRoot = path.join(root, "state"); - const storageRoot = path.join(stateRoot, "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const sourceRoot = path.join(storageRoot, "source"); - const targetRoot = path.join(storageRoot, "target"); - yield* fs.makeDirectory(sourceRoot, { recursive: true }); - yield* fs.makeDirectory(targetRoot, { recursive: true }); - yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ - target: dockerTarget, - root, - }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const makeStorageAt = ( - instanceId: string, - instanceRoot: string, - storageRootValue: string, - cacheRootValue: string, - ) => - makeDockerDatabaseStorage({ + it.live("starts a storage helper with the mirror image selected during preparation", () => { + const engine = fakeHelperEngine(); + return Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "storage-helper-mirror-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "database"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + imageMirrors: () => [helperMirror], + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const storage = yield* makeDockerDatabaseStorage({ runtime: "docker", target: dockerTarget, - stackId: "storage-test", - instanceId, + stackId: "storage-helper-mirror", + instanceId: "database", instanceRoot, - root: storageRootValue, - cacheRoot: cacheRootValue, + root: storageRoot, + cacheRoot, fs, path, crypto, container, spawner, }); - const makeStorage = (instanceId: string, instanceRoot: string) => - makeStorageAt(instanceId, instanceRoot, storageRoot, cacheRoot); - const source = yield* makeStorage("source", sourceRoot); - const target = yield* makeStorage("target", targetRoot); - const ownerScope = yield* Scope.Scope; - yield* Scope.addFinalizer( - ownerScope, - Effect.gen(function* () { - yield* source.destroyData("17").pipe(Effect.ignore); - yield* target.destroyData("17").pipe(Effect.ignore); - const marker = yield* fs - .readFileString(path.join(sourceRoot, ".supabase-database-storage.json")) - .pipe(Effect.option); - if (Option.isSome(marker)) { - const parsed = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - marker.value, - ).pipe(Effect.option); - if (Option.isSome(parsed) && parsed.value.volume !== undefined) - yield* docker(["volume", "rm", parsed.value.volume]).pipe(Effect.ignore); - } - }).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - Effect.ignore, - ), - ); - yield* source.prepare("17"); - const sourceMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), - ); - if (sourceMarker.backend !== "docker" || sourceMarker.volume === undefined) - return yield* new DockerTestError({ message: "Docker test selected host fallback" }); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceMarker.volume},dst=/store`, - helperImage, - "/bin/sh", - "-c", - `set -eu; printf 17 > ${quote(`/store/${sourceMarker.namespace}/data/PG_VERSION`)}; printf source > ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}; /usr/bin/busybox setfattr -n user.storage-smoke -v preserved ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}`, - ]); - yield* source.markInitialized("17"); - yield* fs.writeFileString( - path.join(sourceRoot, ".supabase-database-ready.json"), - '{"version":"17","runtime":"docker","profile":"supabase"}', - ); - yield* source.saveSnapshot("17", "roundtrip"); - - const copiedRoot = path.join(root, "copied-state", "stack", "data", "source"); - const copiedStorageRoot = path.join(root, "copied-state", "stack", "data"); - const copiedCacheRoot = path.join(root, "copied-cache"); - yield* fs.makeDirectory(copiedRoot, { recursive: true }); - yield* fs.makeDirectory(copiedCacheRoot, { recursive: true }); - yield* fs.writeFileString( - path.join(copiedRoot, ".supabase-database-storage.json"), - yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), - ); - const copied = yield* makeStorageAt( - "source", - copiedRoot, - copiedStorageRoot, - copiedCacheRoot, - ); - const mountError = yield* copied.mount("17").pipe(Effect.flip); - expect(mountError.message).toContain("another state directory"); - const removeError = yield* copied.removeData("17").pipe(Effect.flip); - expect(removeError.message).toContain("another state directory"); - const destroyError = yield* copied.destroyData("17").pipe(Effect.flip); - expect(destroyError.message).toContain("another state directory"); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceMarker.volume},dst=/store`, - helperImage, - "/bin/sh", - "-c", - `test "$(cat /store/${sourceMarker.namespace}/data/fixture)" = source`, - ]); - - yield* target.prepare("17"); - expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(true); - const targetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(path.join(targetRoot, ".supabase-database-storage.json")), - ); - expect(targetMarker.volume).toBe(sourceMarker.volume); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceMarker.volume},dst=/store,volume-subpath=${targetMarker.namespace}/data`, - helperImage, - "/bin/sh", - "-c", - `set -eu; test "$(cat /store/fixture)" = source; /usr/bin/busybox setfattr -x user.storage-smoke /store/fixture`, - ]); - yield* target.removeData("unsupported"); - const resetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(path.join(targetRoot, ".supabase-database-storage.json")), - ); - expect(resetMarker.volume).toBe(sourceMarker.volume); - expect(resetMarker.initialized).toBe(false); - - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceMarker.volume},dst=/store`, - helperImage, - "/bin/sh", - "-c", - 'descriptor=$(/usr/bin/busybox find /store -name descriptor.json -print -quit); printf corrupt > "$descriptor"', - ]); - const corrupt = yield* target.restoreSnapshot("17", "roundtrip").pipe( - Effect.matchEffect({ - onFailure: () => Effect.succeed(true), - onSuccess: () => Effect.succeed(false), - }), - ); - expect(corrupt).toBe(true); - yield* source.saveSnapshot("17", "roundtrip"); + yield* storage.prepare("17"); + yield* storage.removeData("17"); - yield* target.restoreSnapshot("17", "roundtrip"); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceMarker.volume},dst=/store`, - helperImage, - "/bin/sh", - "-c", - `rm -rf ${quote(`/store/${targetMarker.namespace}/data`)}`, - ]); - const missing = yield* target.prepare("17").pipe(Effect.exit); - expect(missing).toSatisfy((exit) => Exit.isFailure(exit)); + const run = engine.commands.find((args) => args[0] === "run"); + const shellIndex = run?.indexOf("/bin/sh") ?? -1; + expect(run?.[shellIndex - 1]).toBe(helperMirror); + expect( + engine.commands.filter((args) => args[0] === "pull").map((args) => args.at(-1)), + ).toEqual([expect.stringContaining("ghcr.io/supabase/cli/postgres:"), helperMirror]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); - yield* target.removeData("17"); - yield* source.saveSnapshot("17", "retention-first"); - yield* source.saveSnapshot("17", "retention-second"); - yield* source.saveSnapshot("17", "retention-third"); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceMarker.volume},dst=/store`, - helperImage, - "/bin/sh", - "-c", - `test "$(/usr/bin/busybox find ${quote(`/store/${sourceMarker.cacheNamespace}/entries`)} -mindepth 1 -maxdepth 1 -type d | /usr/bin/busybox wc -l)" -eq 3`, - ]); - expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(false); - yield* source.saveSnapshot("17", "roundtrip"); - yield* source.destroyData("17"); - expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(true); - yield* target.destroyData("17"); - yield* docker(["volume", "rm", sourceMarker.volume]); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); + it.live( + "starts a shared volume helper with the mirror image selected during preparation", + () => { + const engine = fakeHelperEngine(undefined, { volumePresent: true }); + return Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "storage-shared-helper-mirror-", + }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "database"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const stackId = "shared-helper-mirror"; + const instanceId = "database"; + const cachePath = yield* fs.realPath(cacheRoot); + const cacheHash = yield* crypto.digest("SHA-256", new TextEncoder().encode(cachePath)); + const cacheNamespace = `cache-${Array.from(cacheHash, (byte) => + byte.toString(16).padStart(2, "0"), + ) + .join("") + .slice(0, 32)}`; + // Matches the volume name the engine target's own identity resolves, the same way + // DockerDatabaseStorage derives it: a stable digest of the state root and daemon id. + const statePath = yield* fs.realPath(path.join(root, "state")); + const stateHash = yield* crypto.digest( + "SHA-256", + new TextEncoder().encode(`${statePath}\0${dockerTarget.daemonId}`), + ); + const stateDigest = Array.from(stateHash, (byte) => byte.toString(16).padStart(2, "0")) + .join("") + .slice(0, 32); + const marker = yield* Schema.encodeEffect(Schema.fromJsonString(Marker))({ + backend: "docker", + volume: `supabase-db-${stateDigest}`, + namespace: `instance-${stackId}-${instanceId}`, + cacheNamespace, + daemonId: dockerTarget.daemonId, + initialized: false, + }); + yield* fs.writeFileString( + path.join(instanceRoot, ".supabase-database-storage.json"), + marker, + ); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + imageMirrors: () => [helperMirror], + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const helpers = yield* makeDockerHelperRegistry("mirror-test"); + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId, + instanceId, + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + helpers, + }); - it.live("labels its volume with the configured test run", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-test-run-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const instanceRoot = path.join(storageRoot, "database"); - yield* fs.makeDirectory(instanceRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ - target: dockerTarget, - root, - }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const stackId = `storage-test-run-${yield* crypto.randomUUIDv4}`; - const testRunId = `storage-test-run-${(yield* crypto.randomUUIDv4).slice(0, 8)}`; - // This run id overrides the ambient one, so the shared run teardown never sees the volume. - yield* Effect.addFinalizer(() => removeTestRunVolumes(testRunId).pipe(Effect.orDie)); - const storage = yield* makeDockerDatabaseStorage({ - runtime: "docker", - target: dockerTarget, - stackId, - instanceId: "database", - instanceRoot, - root: storageRoot, - cacheRoot, - fs, - path, - crypto, - container, - spawner, - }); - yield* storage - .prepare("17") - .pipe( - Effect.provide( - ConfigProvider.layer( - ConfigProvider.fromEnvRecord({ SUPABASE_STACK_TEST_RUN: testRunId }), - ), - ), - ); - const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(path.join(instanceRoot, ".supabase-database-storage.json")), - ); - if (marker.backend !== "docker" || marker.volume === undefined) - return yield* new DockerTestError({ message: "Docker test selected host fallback" }); - const labels = yield* docker([ - "volume", - "inspect", - "--format", - '{{ index .Labels "com.supabase.stack-test-run" }}', - marker.volume, - ]); - expect(labels).toBe(testRunId); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); + yield* storage.prepare("17"); - it.live("removes an unprepared storage without requiring Docker", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-empty-" }); - const instanceRoot = path.join(root, "state", "stack", "data", "empty"); - const storage = yield* makeDockerDatabaseStorage({ - runtime: "docker", - target: dockerTarget, - stackId: "storage-empty-test", - instanceId: "empty", - instanceRoot, - root: path.join(root, "state", "stack", "data"), - cacheRoot: path.join(root, "cache"), - fs, - path, - crypto, - container: undefined, - spawner, - }); - yield* storage.removeData("17"); - yield* storage.destroyData("17"); - expect(yield* fs.exists(path.join(instanceRoot, ".supabase-database-storage.json"))).toBe( - false, - ); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); + const run = engine.commands.find((args) => args[0] === "run"); + const shellIndex = run?.indexOf("/bin/sh") ?? -1; + expect(run?.[shellIndex - 1]).toBe(helperMirror); + expect( + engine.commands.filter((args) => args[0] === "pull").map((args) => args.at(-1)), + ).toEqual([expect.stringContaining("ghcr.io/supabase/cli/postgres:"), helperMirror]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }, + ); - it.live( - "keeps the prior marker decodable when a real kill lands between staging and publishing an update", - () => + it.live("round-trips PostgreSQL 15 data with its catalog image", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-crash-" }); - const instanceRoot = path.join(root, "state", "stack", "data", "crash"); - const dataRoot = path.join(root, "state", "stack", "data"); + const helperImage = yield* postgresImage("15"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-pg15-" }); + const storageRoot = path.join(root, "state", "stack", "data"); const cacheRoot = path.join(root, "cache"); - yield* fs.makeDirectory(instanceRoot, { recursive: true }); - const markerPath = path.join(instanceRoot, ".supabase-database-storage.json"); - const helperImage = yield* postgresImage("17"); - const storage = yield* makeDockerDatabaseStorage({ - runtime: "docker", + const sourceRoot = path.join(storageRoot, "source"); + const targetRoot = path.join(storageRoot, "target"); + yield* fs.makeDirectory(sourceRoot, { recursive: true }); + yield* fs.makeDirectory(targetRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ target: dockerTarget, - stackId: "storage-crash-test", - instanceId: "crash", - instanceRoot, - root: dataRoot, - cacheRoot, - fs, - path, - crypto, - container: yield* makeContainerRuntime({ - target: dockerTarget, - root: instanceRoot, - }), - spawner, + root, }); - // Removes the real volume the seeding step below creates, rather than leaving it for - // the global test-run teardown, since a container that mounted it may not yet be - // reaped by the time that teardown runs. - yield* Effect.addFinalizer(() => storage.destroyData("17").pipe(Effect.ignore)); - yield* storage.prepare("17"); - const before = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(markerPath), - ); - expect(before.initialized).toBe(false); - if (before.backend !== "docker" || before.volume === undefined) - return yield* new DockerTestError({ message: "Docker test selected host fallback" }); - const volume = before.volume; - // The writer below is killed, not interrupted, so its own one-off helper container (the - // PG_VERSION check's, and any the killed markInitialized retry below starts) never runs - // its own cleanup; removed here instead of leaving it attached when `destroyData` above - // (registered first, so it runs after this) tries to remove the volume. - yield* Effect.addFinalizer(() => - docker(["ps", "--all", "--quiet", "--filter", `volume=${volume}`]).pipe( - Effect.flatMap((listed) => { - const ids = listed - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0); - return ids.length === 0 - ? Effect.void - : docker(["rm", "--force", ...ids]).pipe(Effect.asVoid); - }), + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-pg15-${yield* crypto.randomUUIDv4}`; + const makeStorage = (instanceId: string, instanceRoot: string) => + makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId, + instanceId, + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + const source = yield* makeStorage("source", sourceRoot); + const target = yield* makeStorage("target", targetRoot); + const ownerScope = yield* Scope.Scope; + yield* Scope.addFinalizer( + ownerScope, + Effect.gen(function* () { + yield* source.destroyData("15").pipe(Effect.ignore); + yield* target.destroyData("15").pipe(Effect.ignore); + const marker = yield* fs + .readFileString(path.join(sourceRoot, ".supabase-database-storage.json")) + .pipe(Effect.option); + if (Option.isSome(marker)) { + const parsed = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + marker.value, + ).pipe(Effect.option); + if (Option.isSome(parsed) && parsed.value.volume !== undefined) + yield* engine(["volume", "rm", parsed.value.volume]).pipe(Effect.ignore); + } + }).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), Effect.ignore, ), ); - // `markInitialized` (below, and inside the killed writer) checks this file through a - // helper container; this test is about the marker's own crash-safety, not PostgreSQL's, - // so it is seeded directly rather than through a real database launch. - yield* docker([ + + yield* source.prepare("15"); + const sourceMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), + ); + if (sourceMarker.backend !== "docker" || sourceMarker.volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + yield* engine([ "run", "--rm", "--mount", - `type=volume,src=${before.volume},dst=/store`, + `type=volume,src=${sourceMarker.volume},dst=/store`, helperImage, "/bin/sh", "-c", - `set -eu; mkdir -p ${quote(`/store/${before.namespace}/data`)}; printf 17 > ${quote(`/store/${before.namespace}/data/PG_VERSION`)}`, + `set -eu; printf 15 > ${quote(`/store/${sourceMarker.namespace}/data/PG_VERSION`)}; printf pg15-source > ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}`, ]); - - const writer = yield* spawner.spawn( - ChildProcess.make( - process.execPath, - [dockerStoragePublishLoopFixture, instanceRoot, dataRoot, cacheRoot], - { stdin: "ignore", stdout: "pipe", stderr: "pipe" }, - ), + yield* source.markInitialized("15"); + yield* fs.writeFileString( + path.join(sourceRoot, ".supabase-database-ready.json"), + '{"version":"15","runtime":"docker","profile":"supabase"}', ); - const ready = yield* Deferred.make(); - const stderr = yield* Ref.make(""); - const diagnostics = yield* writer.stderr.pipe( - Stream.decodeText, - Stream.runForEach((chunk) => Ref.update(stderr, (text) => text + chunk)), - Effect.forkScoped, + yield* source.saveSnapshot("15", "restore-me"); + + yield* target.prepare("15"); + expect(yield* target.restoreSnapshot("15", "restore-me")).toBe(true); + const targetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(targetRoot, ".supabase-database-storage.json")), ); - const output = yield* writer.stdout.pipe( - Stream.decodeText, - Stream.splitLines, - Stream.tap((line) => - line === "about-to-publish" ? Deferred.succeed(ready, undefined) : Effect.void, + expect(targetMarker.volume).toBe(sourceMarker.volume); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store,volume-subpath=${targetMarker.namespace}/data`, + helperImage, + "/bin/sh", + "-c", + 'test "$(cat /store/PG_VERSION)" = 15 && test "$(cat /store/fixture)" = pg15-source', + ]); + yield* target.destroyData("15"); + yield* source.destroyData("15"); + yield* engine(["volume", "rm", sourceMarker.volume]); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("round-trips stopped data and protects managed namespaces", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-" }); + const stateRoot = path.join(root, "state"); + const storageRoot = path.join(stateRoot, "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const sourceRoot = path.join(storageRoot, "source"); + const targetRoot = path.join(storageRoot, "target"); + yield* fs.makeDirectory(sourceRoot, { recursive: true }); + yield* fs.makeDirectory(targetRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const makeStorageAt = ( + instanceId: string, + instanceRoot: string, + storageRootValue: string, + cacheRootValue: string, + ) => + makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId: "storage-test", + instanceId, + instanceRoot, + root: storageRootValue, + cacheRoot: cacheRootValue, + fs, + path, + crypto, + container, + spawner, + }); + const makeStorage = (instanceId: string, instanceRoot: string) => + makeStorageAt(instanceId, instanceRoot, storageRoot, cacheRoot); + const source = yield* makeStorage("source", sourceRoot); + const target = yield* makeStorage("target", targetRoot); + const ownerScope = yield* Scope.Scope; + yield* Scope.addFinalizer( + ownerScope, + Effect.gen(function* () { + yield* source.destroyData("17").pipe(Effect.ignore); + yield* target.destroyData("17").pipe(Effect.ignore); + const marker = yield* fs + .readFileString(path.join(sourceRoot, ".supabase-database-storage.json")) + .pipe(Effect.option); + if (Option.isSome(marker)) { + const parsed = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + marker.value, + ).pipe(Effect.option); + if (Option.isSome(parsed) && parsed.value.volume !== undefined) + yield* engine(["volume", "rm", parsed.value.volume]).pipe(Effect.ignore); + } + }).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.ignore, ), - Stream.runDrain, - Effect.forkScoped, ); - const writerFailure = (reason: string) => - Ref.get(stderr).pipe( - Effect.flatMap((text) => - Effect.fail(new PublishBarrierError({ message: `${reason}: ${text}` })), - ), - ); - // The fixture announces this boundary itself, right before the real rename call; - // killing here lands in the staging-to-publish gap on every run, not "most of the time". - yield* Deferred.await(ready).pipe( - Effect.raceFirst( - writer.exitCode.pipe( - Effect.matchEffect({ - onFailure: (cause) => - writerFailure(`Writer exited before readiness: ${String(cause)}`), - onSuccess: (code) => writerFailure(`Writer exited before readiness (${code})`), - }), - ), - ), - Effect.timeoutOrElse({ - duration: "10 seconds", - orElse: () => writerFailure("Writer did not reach the publish boundary"), - }), + yield* source.prepare("17"); + const sourceMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), ); - yield* Effect.sync(() => process.kill(Number(writer.pid), "SIGKILL")); - yield* writer.exitCode.pipe(Effect.ignore); - yield* Fiber.join(output); - yield* Fiber.join(diagnostics); - - // The killed writer never reached the rename: the prior marker is untouched, and only - // its staging file, never a partial target, is left behind. - const afterKill = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(markerPath), + if (sourceMarker.backend !== "docker" || sourceMarker.volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `set -eu; printf 17 > ${quote(`/store/${sourceMarker.namespace}/data/PG_VERSION`)}; printf source > ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}; /usr/bin/busybox setfattr -n user.storage-smoke -v preserved ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}`, + ]); + yield* source.markInitialized("17"); + yield* fs.writeFileString( + path.join(sourceRoot, ".supabase-database-ready.json"), + '{"version":"17","runtime":"docker","profile":"supabase"}', ); - expect(afterKill).toEqual(before); - expect( - (yield* fs.readDirectory(instanceRoot)).filter((entry) => entry.endsWith(".tmp")), - ).toHaveLength(1); + yield* source.saveSnapshot("17", "roundtrip"); - yield* storage.markInitialized("17"); - const afterRetry = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(markerPath), + const copiedRoot = path.join(root, "copied-state", "stack", "data", "source"); + const copiedStorageRoot = path.join(root, "copied-state", "stack", "data"); + const copiedCacheRoot = path.join(root, "copied-cache"); + yield* fs.makeDirectory(copiedRoot, { recursive: true }); + yield* fs.makeDirectory(copiedCacheRoot, { recursive: true }); + yield* fs.writeFileString( + path.join(copiedRoot, ".supabase-database-storage.json"), + yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), ); - expect(afterRetry.initialized).toBe(true); - }), - ).pipe(Effect.provide(NodeServices.layer)), - 20_000, - ); - - it.live("refuses to start unmarked data and removes it through the helper on destroy", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const helperImage = yield* postgresImage("17"); - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-unmarked-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const instanceRoot = path.join(storageRoot, "unmarked"); - const dataRoot = path.join(instanceRoot, "data"); - const checkpointsRoot = path.join(instanceRoot, ".supabase-snapshots"); - yield* fs.makeDirectory(path.join(dataRoot, "base"), { recursive: true }); - yield* fs.writeFileString(path.join(dataRoot, "base", "fixture"), "unmarked"); - yield* fs.makeDirectory(path.join(checkpointsRoot, "entries", "checkpoint", "data"), { - recursive: true, - }); - yield* fs.writeFileString( - path.join(checkpointsRoot, "entries", "checkpoint", "data", "PG_VERSION"), - "17", - ); - yield* docker([ - "run", - "--rm", - "--mount", - `type=bind,src=${instanceRoot},dst=/instance`, - helperImage, - "/bin/sh", - "-c", - "chown -R 100:101 /instance/data /instance/.supabase-snapshots; chmod -R 700 /instance/data /instance/.supabase-snapshots", - ]); - const container = yield* makeContainerRuntime({ - target: dockerTarget, - root, - }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const storage = yield* makeDockerDatabaseStorage({ - runtime: "docker", - target: dockerTarget, - stackId: `storage-unmarked-${yield* crypto.randomUUIDv4}`, - instanceId: "unmarked", - instanceRoot, - root: storageRoot, - cacheRoot, - fs, - path, - crypto, - container, - spawner, - }); - const markerPath = path.join(instanceRoot, ".supabase-database-storage.json"); + const copied = yield* makeStorageAt( + "source", + copiedRoot, + copiedStorageRoot, + copiedCacheRoot, + ); + const mountError = yield* copied.mount("17").pipe(Effect.flip); + expect(mountError.message).toContain("another state directory"); + const removeError = yield* copied.removeData("17").pipe(Effect.flip); + expect(removeError.message).toContain("another state directory"); + const destroyError = yield* copied.destroyData("17").pipe(Effect.flip); + expect(destroyError.message).toContain("another state directory"); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `test "$(cat /store/${sourceMarker.namespace}/data/fixture)" = source`, + ]); - const failure = yield* storage.prepare("17").pipe(Effect.flip); - expect(failure.message).toContain("without a storage marker"); - expect(yield* fs.exists(markerPath)).toBe(false); + yield* target.prepare("17"); + expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(true); + const targetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(targetRoot, ".supabase-database-storage.json")), + ); + expect(targetMarker.volume).toBe(sourceMarker.volume); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store,volume-subpath=${targetMarker.namespace}/data`, + helperImage, + "/bin/sh", + "-c", + `set -eu; test "$(cat /store/fixture)" = source; /usr/bin/busybox setfattr -x user.storage-smoke /store/fixture`, + ]); - yield* storage.destroyData("17"); - expect(yield* fs.exists(dataRoot)).toBe(false); - expect(yield* fs.exists(checkpointsRoot)).toBe(false); - expect(yield* fs.exists(markerPath)).toBe(false); - yield* fs.remove(cacheRoot, { recursive: true, force: true }); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); + yield* target.removeData("unsupported"); + const resetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(targetRoot, ".supabase-database-storage.json")), + ); + expect(resetMarker.volume).toBe(sourceMarker.volume); + expect(resetMarker.initialized).toBe(false); - it.live("refuses to adopt unmarked Docker data at startup", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-unmarked-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const instanceRoot = path.join(storageRoot, "unmarked"); - const dataRoot = path.join(instanceRoot, "data"); - yield* fs.makeDirectory(dataRoot, { recursive: true }); - yield* fs.writeFileString(path.join(dataRoot, "PG_VERSION"), "17\n"); - const storage = yield* makeDockerDatabaseStorage({ - runtime: "docker", - target: dockerTarget, - stackId: "storage-docker-unmarked", - instanceId: "unmarked", - instanceRoot, - root: storageRoot, - cacheRoot: path.join(root, "cache"), - fs, - path, - crypto, - container: yield* makeContainerRuntime({ - target: dockerTarget, - root, - }), - spawner: yield* ChildProcessSpawner.ChildProcessSpawner, - }); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + 'descriptor=$(/usr/bin/busybox find /store -name descriptor.json -print -quit); printf corrupt > "$descriptor"', + ]); + const corrupt = yield* target.restoreSnapshot("17", "roundtrip").pipe( + Effect.matchEffect({ + onFailure: () => Effect.succeed(true), + onSuccess: () => Effect.succeed(false), + }), + ); + expect(corrupt).toBe(true); + yield* source.saveSnapshot("17", "roundtrip"); - const failure = yield* storage.prepare("17").pipe(Effect.flip); - expect(failure.message).toContain("without a storage marker"); - expect(yield* fs.exists(path.join(instanceRoot, ".supabase-database-storage.json"))).toBe( - false, - ); - expect(yield* fs.readFileString(path.join(dataRoot, "PG_VERSION"))).toBe("17\n"); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); + yield* target.restoreSnapshot("17", "roundtrip"); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `rm -rf ${quote(`/store/${targetMarker.namespace}/data`)}`, + ]); + const missing = yield* target.prepare("17").pipe(Effect.exit); + expect(missing).toSatisfy((exit) => Exit.isFailure(exit)); - it.live("recovers after an owned helper is removed externally", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-helper-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const instanceRoot = path.join(storageRoot, "recovery"); - yield* fs.makeDirectory(instanceRoot, { recursive: true }); - yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ - target: dockerTarget, - root, - }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const stackId = `storage-helper-recovery-${yield* crypto.randomUUIDv4}`; - const instanceId = "recovery"; - const helperScope = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(helperScope, Exit.void)); - const helperRegistry = yield* makeDockerHelperRegistry(yield* crypto.randomUUIDv4).pipe( - Effect.provideService(Scope.Scope, helperScope), - ); - const storage = yield* makeDockerDatabaseStorage({ - runtime: "docker", - target: dockerTarget, - stackId, - instanceId, - instanceRoot, - root: storageRoot, - cacheRoot, - fs, - path, - crypto, - container, - spawner, - helpers: helperRegistry, - }); - yield* storage.prepare("17"); - const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(path.join(instanceRoot, ".supabase-database-storage.json")), - ); - const volume = marker.volume; - if (volume === undefined) - return yield* new DockerTestError({ message: "Docker storage volume missing" }); - const helpers = yield* docker([ - "ps", - "--filter", - "label=com.supabase.stack=" + stackId, - "--format", - "{{.Names}}", - ]); - const helper = helpers.split("\n").find((name) => name.length > 0); - if (helper === undefined) - return yield* new DockerTestError({ message: "Owned helper was not discoverable" }); - yield* docker(["rm", "-f", helper]); - yield* storage.prepare("17"); - yield* storage.destroyData("17"); - yield* Scope.close(helperScope, Exit.void); - yield* docker(["volume", "rm", volume]); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); + yield* target.removeData("17"); + yield* source.saveSnapshot("17", "retention-first"); + yield* source.saveSnapshot("17", "retention-second"); + yield* source.saveSnapshot("17", "retention-third"); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `test "$(/usr/bin/busybox find ${quote(`/store/${sourceMarker.cacheNamespace}/entries`)} -mindepth 1 -maxdepth 1 -type d | /usr/bin/busybox wc -l)" -eq 3`, + ]); + expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(false); + yield* source.saveSnapshot("17", "roundtrip"); + yield* source.destroyData("17"); + expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(true); + yield* target.destroyData("17"); + yield* engine(["volume", "rm", sourceMarker.volume]); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); - it.live("keeps another owner's volume helper running when one owner closes", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-helper-owner-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const instanceRoot = path.join(storageRoot, "database"); - yield* fs.makeDirectory(instanceRoot, { recursive: true }); - yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ - target: dockerTarget, - root, - }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const stackId = `storage-helper-owner-${yield* crypto.randomUUIDv4}`; - const firstScope = yield* Scope.make(); - const secondScope = yield* Scope.make(); - yield* Effect.addFinalizer(() => - Effect.gen(function* () { - yield* Scope.close(secondScope, Exit.void); - yield* Scope.close(firstScope, Exit.void); - const markerText = yield* fs - .readFileString(path.join(instanceRoot, ".supabase-database-storage.json")) - .pipe(Effect.option); - if (Option.isNone(markerText)) return; - const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - markerText.value, - ).pipe(Effect.option); - if (Option.isSome(marker) && marker.value.volume !== undefined) - yield* docker(["volume", "rm", marker.value.volume]).pipe(Effect.ignore); - }).pipe(Effect.ignore), - ); - const firstHelpers = yield* makeDockerHelperRegistry("first-owner").pipe( - Effect.provideService(Scope.Scope, firstScope), - ); - const secondHelpers = yield* makeDockerHelperRegistry("second-owner").pipe( - Effect.provideService(Scope.Scope, secondScope), - ); - const makeStorage = (helpers: DockerHelperRegistry) => - makeDockerDatabaseStorage({ + it.live("labels its volume with the configured test run", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-test-run-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "database"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-test-run-${yield* crypto.randomUUIDv4}`; + const testRunId = `storage-test-run-${(yield* crypto.randomUUIDv4).slice(0, 8)}`; + // This run id overrides the ambient one, so the shared run teardown never sees the volume. + yield* Effect.addFinalizer(() => removeTestRunVolumes(testRunId).pipe(Effect.orDie)); + const storage = yield* makeDockerDatabaseStorage({ runtime: "docker", target: dockerTarget, stackId, @@ -1122,142 +703,329 @@ describe("Docker database storage", { timeout: 120_000 }, () => { crypto, container, spawner, - helpers, }); - const firstStorage = yield* makeStorage(firstHelpers).pipe( - Effect.provideService(Scope.Scope, firstScope), - ); - const secondStorage = yield* makeStorage(secondHelpers).pipe( - Effect.provideService(Scope.Scope, secondScope), - ); - yield* firstStorage.prepare("17"); - yield* secondStorage.prepare("17"); - const runningHelpers = (yield* docker([ - "ps", - "--filter", - `label=com.supabase.stack=${stackId}`, - "--format", - "{{.Names}}", - ])) - .split("\n") - .filter((name) => name.length > 0); - expect(runningHelpers).toHaveLength(2); - - yield* Scope.close(firstScope, Exit.void); - const remainingHelpers = (yield* docker([ - "ps", - "--filter", - `label=com.supabase.stack=${stackId}`, - "--format", - "{{.Names}}", - ])) - .split("\n") - .filter((name) => name.length > 0); - expect(remainingHelpers).toHaveLength(1); - expect(runningHelpers).toContain(remainingHelpers[0]); - yield* secondStorage.prepare("17"); - yield* secondStorage.destroyData("17"); - yield* Scope.close(secondScope, Exit.void); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); + yield* storage + .prepare("17") + .pipe( + Effect.provide( + ConfigProvider.layer( + ConfigProvider.fromEnvRecord({ SUPABASE_STACK_TEST_RUN: testRunId }), + ), + ), + ); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(instanceRoot, ".supabase-database-storage.json")), + ); + if (marker.backend !== "docker" || marker.volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + const labels = yield* engine([ + "volume", + "inspect", + "--format", + '{{ index .Labels "com.supabase.stack-test-run" }}', + marker.volume, + ]); + expect(labels).toBe(testRunId); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); - it.live("destroys an initialized storage after its volume is removed", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const helperImage = yield* postgresImage("17"); - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-missing-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const instanceRoot = path.join(storageRoot, "missing"); - yield* fs.makeDirectory(instanceRoot, { recursive: true }); - yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ - target: dockerTarget, - root, - }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const stackId = `storage-missing-${yield* crypto.randomUUIDv4}`; - const makeStorage = () => - makeDockerDatabaseStorage({ + it.live("removes an unprepared storage without requiring Docker", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-empty-" }); + const instanceRoot = path.join(root, "state", "stack", "data", "empty"); + const storage = yield* makeDockerDatabaseStorage({ runtime: "docker", target: dockerTarget, - stackId, - instanceId: "missing", + stackId: "storage-empty-test", + instanceId: "empty", instanceRoot, - root: storageRoot, - cacheRoot, + root: path.join(root, "state", "stack", "data"), + cacheRoot: path.join(root, "cache"), fs, path, crypto, - container, + container: undefined, spawner, }); - let volume: string | undefined; - yield* Effect.scoped( + yield* storage.removeData("17"); + yield* storage.destroyData("17"); + expect(yield* fs.exists(path.join(instanceRoot, ".supabase-database-storage.json"))).toBe( + false, + ); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live( + "keeps the prior marker decodable when a real kill lands between staging and publishing an update", + () => + Effect.scoped( Effect.gen(function* () { - const storage = yield* makeStorage(); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-crash-" }); + const instanceRoot = path.join(root, "state", "stack", "data", "crash"); + const dataRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + const markerPath = path.join(instanceRoot, ".supabase-database-storage.json"); + const helperImage = yield* postgresImage("17"); + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId: "storage-crash-test", + instanceId: "crash", + instanceRoot, + root: dataRoot, + cacheRoot, + fs, + path, + crypto, + container: yield* makeContainerRuntime({ + target: dockerTarget, + root: instanceRoot, + }), + spawner, + }); + // Removes the real volume the seeding step below creates, rather than leaving it for + // the global test-run teardown, since a container that mounted it may not yet be + // reaped by the time that teardown runs. + yield* Effect.addFinalizer(() => storage.destroyData("17").pipe(Effect.ignore)); yield* storage.prepare("17"); - const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(path.join(instanceRoot, ".supabase-database-storage.json")), + const before = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(markerPath), ); - volume = marker.volume; - if (volume === undefined) - return yield* new DockerTestError({ message: "Docker storage volume missing" }); - const mount = yield* storage.mount("17"); - if (mount.type !== "volume" || mount.volumeSubpath === undefined) - return yield* new DockerTestError({ message: "Docker volume backend unavailable" }); - yield* docker([ + expect(before.initialized).toBe(false); + if (before.backend !== "docker" || before.volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + const volume = before.volume; + // The writer below is killed, not interrupted, so its own one-off helper container (the + // PG_VERSION check's, and any the killed markInitialized retry below starts) never runs + // its own cleanup; removed here instead of leaving it attached when `destroyData` above + // (registered first, so it runs after this) tries to remove the volume. + yield* Effect.addFinalizer(() => + engine(["ps", "--all", "--quiet", "--filter", `volume=${volume}`]).pipe( + Effect.flatMap((listed) => { + const ids = listed + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); + return ids.length === 0 + ? Effect.void + : engine(["rm", "--force", ...ids]).pipe(Effect.asVoid); + }), + Effect.ignore, + ), + ); + // `markInitialized` (below, and inside the killed writer) checks this file through a + // helper container; this test is about the marker's own crash-safety, not PostgreSQL's, + // so it is seeded directly rather than through a real database launch. + yield* engine([ "run", "--rm", "--mount", - `type=volume,src=${volume},dst=/data,volume-subpath=${mount.volumeSubpath}`, + `type=volume,src=${before.volume},dst=/store`, helperImage, "/bin/sh", "-c", - "printf 17 > /data/PG_VERSION", + `set -eu; mkdir -p ${quote(`/store/${before.namespace}/data`)}; printf 17 > ${quote(`/store/${before.namespace}/data/PG_VERSION`)}`, ]); - yield* storage.markInitialized("17"); - }), - ); - if (volume === undefined) return yield* new DockerTestError({ message: "Missing volume" }); - const removedVolume = volume; - yield* docker(["volume", "rm", removedVolume]); - yield* Effect.scoped( - Effect.gen(function* () { - const storage = yield* makeStorage(); - const prepare = yield* storage.prepare("17").pipe(Effect.exit); - expect(prepare).toSatisfy((exit) => Exit.isFailure(exit)); - const removeError = yield* storage.removeData("17").pipe(Effect.flip); - expect(removeError.message).toMatch(/no such volume/iu); - expect(yield* docker(["volume", "inspect", removedVolume]).pipe(Effect.exit)).toSatisfy( - (exit) => Exit.isFailure(exit), + + const writer = yield* spawner.spawn( + ChildProcess.make( + process.execPath, + [dockerStoragePublishLoopFixture, instanceRoot, dataRoot, cacheRoot], + { stdin: "ignore", stdout: "pipe", stderr: "pipe" }, + ), + ); + const ready = yield* Deferred.make(); + const stderr = yield* Ref.make(""); + const diagnostics = yield* writer.stderr.pipe( + Stream.decodeText, + Stream.runForEach((chunk) => Ref.update(stderr, (text) => text + chunk)), + Effect.forkScoped, + ); + const output = yield* writer.stdout.pipe( + Stream.decodeText, + Stream.splitLines, + Stream.tap((line) => + line === "about-to-publish" ? Deferred.succeed(ready, undefined) : Effect.void, + ), + Stream.runDrain, + Effect.forkScoped, + ); + const writerFailure = (reason: string) => + Ref.get(stderr).pipe( + Effect.flatMap((text) => + Effect.fail(new PublishBarrierError({ message: `${reason}: ${text}` })), + ), + ); + // The fixture announces this boundary itself, right before the real rename call; + // killing here lands in the staging-to-publish gap on every run, not "most of the time". + yield* Deferred.await(ready).pipe( + Effect.raceFirst( + writer.exitCode.pipe( + Effect.matchEffect({ + onFailure: (cause) => + writerFailure(`Writer exited before readiness: ${String(cause)}`), + onSuccess: (code) => writerFailure(`Writer exited before readiness (${code})`), + }), + ), + ), + Effect.timeoutOrElse({ + duration: "10 seconds", + orElse: () => writerFailure("Writer did not reach the publish boundary"), + }), ); - expect(yield* storage.destroyData("17").pipe(Effect.exit)).toSatisfy((exit) => - Exit.isSuccess(exit), + yield* Effect.sync(() => process.kill(Number(writer.pid), "SIGKILL")); + yield* writer.exitCode.pipe(Effect.ignore); + yield* Fiber.join(output); + yield* Fiber.join(diagnostics); + + // The killed writer never reached the rename: the prior marker is untouched, and only + // its staging file, never a partial target, is left behind. + const afterKill = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(markerPath), + ); + expect(afterKill).toEqual(before); + expect( + (yield* fs.readDirectory(instanceRoot)).filter((entry) => entry.endsWith(".tmp")), + ).toHaveLength(1); + + yield* storage.markInitialized("17"); + const afterRetry = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(markerPath), ); + expect(afterRetry.initialized).toBe(true); }), - ); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); + ).pipe(Effect.provide(NodeServices.layer)), + 20_000, + ); + + it.live("refuses to start unmarked data and removes it through the helper on destroy", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-unmarked-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "unmarked"); + const dataRoot = path.join(instanceRoot, "data"); + const checkpointsRoot = path.join(instanceRoot, ".supabase-snapshots"); + yield* fs.makeDirectory(path.join(dataRoot, "base"), { recursive: true }); + yield* fs.writeFileString(path.join(dataRoot, "base", "fixture"), "unmarked"); + yield* fs.makeDirectory(path.join(checkpointsRoot, "entries", "checkpoint", "data"), { + recursive: true, + }); + yield* fs.writeFileString( + path.join(checkpointsRoot, "entries", "checkpoint", "data", "PG_VERSION"), + "17", + ); + yield* engine([ + "run", + "--rm", + "--mount", + `type=bind,src=${instanceRoot},dst=/instance`, + helperImage, + "/bin/sh", + "-c", + "chown -R 100:101 /instance/data /instance/.supabase-snapshots; chmod -R 700 /instance/data /instance/.supabase-snapshots", + ]); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId: `storage-unmarked-${yield* crypto.randomUUIDv4}`, + instanceId: "unmarked", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + const markerPath = path.join(instanceRoot, ".supabase-database-storage.json"); + + const failure = yield* storage.prepare("17").pipe(Effect.flip); + expect(failure.message).toContain("without a storage marker"); + expect(yield* fs.exists(markerPath)).toBe(false); + + yield* storage.destroyData("17"); + expect(yield* fs.exists(dataRoot)).toBe(false); + expect(yield* fs.exists(checkpointsRoot)).toBe(false); + expect(yield* fs.exists(markerPath)).toBe(false); + yield* fs.remove(cacheRoot, { recursive: true, force: true }); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("refuses to adopt unmarked Docker data at startup", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-unmarked-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const instanceRoot = path.join(storageRoot, "unmarked"); + const dataRoot = path.join(instanceRoot, "data"); + yield* fs.makeDirectory(dataRoot, { recursive: true }); + yield* fs.writeFileString(path.join(dataRoot, "PG_VERSION"), "17\n"); + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId: "storage-docker-unmarked", + instanceId: "unmarked", + instanceRoot, + root: storageRoot, + cacheRoot: path.join(root, "cache"), + fs, + path, + crypto, + container: yield* makeContainerRuntime({ + target: dockerTarget, + root, + }), + spawner: yield* ChildProcessSpawner.ChildProcessSpawner, + }); + + const failure = yield* storage.prepare("17").pipe(Effect.flip); + expect(failure.message).toContain("without a storage marker"); + expect(yield* fs.exists(path.join(instanceRoot, ".supabase-database-storage.json"))).toBe( + false, + ); + expect(yield* fs.readFileString(path.join(dataRoot, "PG_VERSION"))).toBe("17\n"); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); - it.live( - "destroys the owned namespace in the shared volume after its on-disk marker is deleted, using the identity resolved in memory", - () => + it.live("recovers after an owned helper is removed externally", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; - const helperImage = yield* postgresImage("17"); - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-no-marker-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-helper-" }); const storageRoot = path.join(root, "state", "stack", "data"); const cacheRoot = path.join(root, "cache"); - const instanceRoot = path.join(storageRoot, "sleeping"); + const instanceRoot = path.join(storageRoot, "recovery"); yield* fs.makeDirectory(instanceRoot, { recursive: true }); yield* fs.makeDirectory(cacheRoot, { recursive: true }); const container = yield* makeContainerRuntime({ @@ -1265,14 +1033,18 @@ describe("Docker database storage", { timeout: 120_000 }, () => { root, }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const stackId = `storage-no-marker-${yield* crypto.randomUUIDv4}`; - // One storage object spans the whole case, the way an Owner keeps one per instance for - // the stack's lifetime: its resolved identity stays in memory across the sleep below. + const stackId = `storage-helper-recovery-${yield* crypto.randomUUIDv4}`; + const instanceId = "recovery"; + const helperScope = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(helperScope, Exit.void)); + const helperRegistry = yield* makeDockerHelperRegistry(yield* crypto.randomUUIDv4).pipe( + Effect.provideService(Scope.Scope, helperScope), + ); const storage = yield* makeDockerDatabaseStorage({ runtime: "docker", target: dockerTarget, stackId, - instanceId: "sleeping", + instanceId, instanceRoot, root: storageRoot, cacheRoot, @@ -1281,68 +1053,46 @@ describe("Docker database storage", { timeout: 120_000 }, () => { crypto, container, spawner, + helpers: helperRegistry, }); yield* storage.prepare("17"); - const markerPath = path.join(instanceRoot, ".supabase-database-storage.json"); const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(markerPath), + yield* fs.readFileString(path.join(instanceRoot, ".supabase-database-storage.json")), ); - if (marker.backend !== "docker" || marker.volume === undefined) - return yield* new DockerTestError({ message: "Docker test selected host fallback" }); const volume = marker.volume; - const mount = yield* storage.mount("17"); - if (mount.type !== "volume" || mount.volumeSubpath === undefined) - return yield* new DockerTestError({ message: "Docker volume backend unavailable" }); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${volume},dst=/data,volume-subpath=${mount.volumeSubpath}`, - helperImage, - "/bin/sh", - "-c", - "printf 17 > /data/PG_VERSION", + if (volume === undefined) + return yield* new DockerTestError({ message: "Docker storage volume missing" }); + const helpers = yield* engine([ + "ps", + "--filter", + "label=com.supabase.stack=" + stackId, + "--format", + "{{.Names}}", ]); - yield* storage.markInitialized("17"); - - // The database sleeps (the registered instance stays stopped); its storage object keeps - // running, so this is the only change: the registration's on-disk marker is now gone. - yield* fs.remove(markerPath); - + const helper = helpers.split("\n").find((name) => name.length > 0); + if (helper === undefined) + return yield* new DockerTestError({ message: "Owned helper was not discoverable" }); + yield* engine(["rm", "-f", helper]); + yield* storage.prepare("17"); yield* storage.destroyData("17"); - - const remaining = yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${volume},dst=/store`, - helperImage, - "/bin/sh", - "-c", - `[ -e ${quote(`/store/${marker.namespace}`)} ] && echo present || echo absent`, - ]); - expect(remaining.trim()).toBe("absent"); - yield* docker(["volume", "rm", volume]); + yield* Scope.close(helperScope, Exit.void); + yield* engine(["volume", "rm", volume]); }), ).pipe(Effect.provide(NodeServices.layer)), - ); + ); - it.live( - "destroys the owned namespace after its whole state root is deleted, with no filesystem dependency for an identity already resolved in memory", - () => + it.live("keeps another owner's volume helper running when one owner closes", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; - const helperImage = yield* postgresImage("17"); - // `root` is deleted entirely below; nest it under the scope's own temp dir so the test - // framework's own cleanup of that outer, still-present directory does not fail. - const tempDir = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-no-root-" }); - const root = path.join(tempDir, "state-root"); + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "docker-storage-helper-owner-", + }); const storageRoot = path.join(root, "state", "stack", "data"); const cacheRoot = path.join(root, "cache"); - const instanceRoot = path.join(storageRoot, "sleeping"); + const instanceRoot = path.join(storageRoot, "database"); yield* fs.makeDirectory(instanceRoot, { recursive: true }); yield* fs.makeDirectory(cacheRoot, { recursive: true }); const container = yield* makeContainerRuntime({ @@ -1350,68 +1100,468 @@ describe("Docker database storage", { timeout: 120_000 }, () => { root, }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const stackId = `storage-no-root-${yield* crypto.randomUUIDv4}`; - // One storage object spans the whole case, the way an Owner keeps one per instance for - // the stack's lifetime: its resolved identity stays in memory across the sleep below. - const storage = yield* makeDockerDatabaseStorage({ - runtime: "docker", + const stackId = `storage-helper-owner-${yield* crypto.randomUUIDv4}`; + const firstScope = yield* Scope.make(); + const secondScope = yield* Scope.make(); + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + yield* Scope.close(secondScope, Exit.void); + yield* Scope.close(firstScope, Exit.void); + const markerText = yield* fs + .readFileString(path.join(instanceRoot, ".supabase-database-storage.json")) + .pipe(Effect.option); + if (Option.isNone(markerText)) return; + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + markerText.value, + ).pipe(Effect.option); + if (Option.isSome(marker) && marker.value.volume !== undefined) + yield* engine(["volume", "rm", marker.value.volume]).pipe(Effect.ignore); + }).pipe(Effect.ignore), + ); + const firstHelpers = yield* makeDockerHelperRegistry("first-owner").pipe( + Effect.provideService(Scope.Scope, firstScope), + ); + const secondHelpers = yield* makeDockerHelperRegistry("second-owner").pipe( + Effect.provideService(Scope.Scope, secondScope), + ); + const makeStorage = (helpers: DockerHelperRegistry) => + makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId, + instanceId: "database", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + helpers, + }); + const firstStorage = yield* makeStorage(firstHelpers).pipe( + Effect.provideService(Scope.Scope, firstScope), + ); + const secondStorage = yield* makeStorage(secondHelpers).pipe( + Effect.provideService(Scope.Scope, secondScope), + ); + yield* firstStorage.prepare("17"); + yield* secondStorage.prepare("17"); + const runningHelpers = (yield* engine([ + "ps", + "--filter", + `label=com.supabase.stack=${stackId}`, + "--format", + "{{.Names}}", + ])) + .split("\n") + .filter((name) => name.length > 0); + expect(runningHelpers).toHaveLength(2); + + yield* Scope.close(firstScope, Exit.void); + const remainingHelpers = (yield* engine([ + "ps", + "--filter", + `label=com.supabase.stack=${stackId}`, + "--format", + "{{.Names}}", + ])) + .split("\n") + .filter((name) => name.length > 0); + expect(remainingHelpers).toHaveLength(1); + expect(runningHelpers).toContain(remainingHelpers[0]); + yield* secondStorage.prepare("17"); + yield* secondStorage.destroyData("17"); + yield* Scope.close(secondScope, Exit.void); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("destroys an initialized storage after its volume is removed", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-missing-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "missing"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ target: dockerTarget, - stackId, - instanceId: "sleeping", - instanceRoot, - root: storageRoot, - cacheRoot, - fs, - path, - crypto, - container, - spawner, + root, }); - yield* storage.prepare("17"); - const markerPath = path.join(instanceRoot, ".supabase-database-storage.json"); - const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(markerPath), + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-missing-${yield* crypto.randomUUIDv4}`; + const makeStorage = () => + makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId, + instanceId: "missing", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + let volume: string | undefined; + yield* Effect.scoped( + Effect.gen(function* () { + const storage = yield* makeStorage(); + yield* storage.prepare("17"); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString( + path.join(instanceRoot, ".supabase-database-storage.json"), + ), + ); + volume = marker.volume; + if (volume === undefined) + return yield* new DockerTestError({ message: "Docker storage volume missing" }); + const mount = yield* storage.mount("17"); + if (mount.type !== "volume" || mount.volumeSubpath === undefined) + return yield* new DockerTestError({ message: "Docker volume backend unavailable" }); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${volume},dst=/data,volume-subpath=${mount.volumeSubpath}`, + helperImage, + "/bin/sh", + "-c", + "printf 17 > /data/PG_VERSION", + ]); + yield* storage.markInitialized("17"); + }), ); - if (marker.backend !== "docker" || marker.volume === undefined) - return yield* new DockerTestError({ message: "Docker test selected host fallback" }); - const volume = marker.volume; - const mount = yield* storage.mount("17"); - if (mount.type !== "volume" || mount.volumeSubpath === undefined) - return yield* new DockerTestError({ message: "Docker volume backend unavailable" }); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${volume},dst=/data,volume-subpath=${mount.volumeSubpath}`, - helperImage, - "/bin/sh", - "-c", - "printf 17 > /data/PG_VERSION", - ]); - yield* storage.markInitialized("17"); + if (volume === undefined) + return yield* new DockerTestError({ message: "Missing volume" }); + const removedVolume = volume; + yield* engine(["volume", "rm", removedVolume]); + yield* Effect.scoped( + Effect.gen(function* () { + const storage = yield* makeStorage(); + const prepare = yield* storage.prepare("17").pipe(Effect.exit); + expect(prepare).toSatisfy((exit) => Exit.isFailure(exit)); + const removeError = yield* storage.removeData("17").pipe(Effect.flip); + expect(removeError.message).toMatch(/no such volume/iu); + expect( + yield* engine(["volume", "inspect", removedVolume]).pipe(Effect.exit), + ).toSatisfy((exit) => Exit.isFailure(exit)); + expect(yield* storage.destroyData("17").pipe(Effect.exit)).toSatisfy((exit) => + Exit.isSuccess(exit), + ); + }), + ); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); - // The database sleeps; its storage object keeps running (the resolved identity is - // already in memory), then the whole state root disappears from under the live owner. - yield* fs.remove(root, { recursive: true, force: true }); + it.live( + "destroys the owned namespace in the shared volume after its on-disk marker is deleted, using the identity resolved in memory", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-no-marker-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "sleeping"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-no-marker-${yield* crypto.randomUUIDv4}`; + // One storage object spans the whole case, the way an Owner keeps one per instance for + // the stack's lifetime: its resolved identity stays in memory across the sleep below. + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId, + instanceId: "sleeping", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + yield* storage.prepare("17"); + const markerPath = path.join(instanceRoot, ".supabase-database-storage.json"); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(markerPath), + ); + if (marker.backend !== "docker" || marker.volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + const volume = marker.volume; + const mount = yield* storage.mount("17"); + if (mount.type !== "volume" || mount.volumeSubpath === undefined) + return yield* new DockerTestError({ message: "Docker volume backend unavailable" }); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${volume},dst=/data,volume-subpath=${mount.volumeSubpath}`, + helperImage, + "/bin/sh", + "-c", + "printf 17 > /data/PG_VERSION", + ]); + yield* storage.markInitialized("17"); - yield* storage.destroyData("17"); + // The database sleeps (the registered instance stays stopped); its storage object keeps + // running, so this is the only change: the registration's on-disk marker is now gone. + yield* fs.remove(markerPath); - const remaining = yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${volume},dst=/store`, - helperImage, - "/bin/sh", - "-c", - `[ -e ${quote(`/store/${marker.namespace}`)} ] && echo present || echo absent`, - ]); - expect(remaining.trim()).toBe("absent"); - yield* docker(["volume", "rm", volume]); + yield* storage.destroyData("17"); + + const remaining = yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `[ -e ${quote(`/store/${marker.namespace}`)} ] && echo present || echo absent`, + ]); + expect(remaining.trim()).toBe("absent"); + yield* engine(["volume", "rm", volume]); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live( + "destroys the owned namespace after its whole state root is deleted, with no filesystem dependency for an identity already resolved in memory", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); + // `root` is deleted entirely below; nest it under the scope's own temp dir so the test + // framework's own cleanup of that outer, still-present directory does not fail. + const tempDir = yield* fs.makeTempDirectoryScoped({ + prefix: "docker-storage-no-root-", + }); + const root = path.join(tempDir, "state-root"); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "sleeping"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-no-root-${yield* crypto.randomUUIDv4}`; + // One storage object spans the whole case, the way an Owner keeps one per instance for + // the stack's lifetime: its resolved identity stays in memory across the sleep below. + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId, + instanceId: "sleeping", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + yield* storage.prepare("17"); + const markerPath = path.join(instanceRoot, ".supabase-database-storage.json"); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(markerPath), + ); + if (marker.backend !== "docker" || marker.volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + const volume = marker.volume; + const mount = yield* storage.mount("17"); + if (mount.type !== "volume" || mount.volumeSubpath === undefined) + return yield* new DockerTestError({ message: "Docker volume backend unavailable" }); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${volume},dst=/data,volume-subpath=${mount.volumeSubpath}`, + helperImage, + "/bin/sh", + "-c", + "printf 17 > /data/PG_VERSION", + ]); + yield* storage.markInitialized("17"); + + // The database sleeps; its storage object keeps running (the resolved identity is + // already in memory), then the whole state root disappears from under the live owner. + yield* fs.remove(root, { recursive: true, force: true }); + + yield* storage.destroyData("17"); + + const remaining = yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `[ -e ${quote(`/store/${marker.namespace}`)} ] && echo present || echo absent`, + ]); + expect(remaining.trim()).toBe("absent"); + yield* engine(["volume", "rm", volume]); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("reopens managed storage and preserves snapshots after source destruction", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-reopen-" }); + const stateRoot = path.join(root, "state"); + const storageRoot = path.join(stateRoot, "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const sourceRoot = path.join(storageRoot, "source"); + const targetRoot = path.join(storageRoot, "target"); + yield* fs.makeDirectory(sourceRoot, { recursive: true }); + yield* fs.makeDirectory(targetRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const makeStorageWithCache = ( + instanceId: string, + instanceRoot: string, + cacheRootValue: string, + ) => + makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId: "storage-reopen-test", + instanceId, + instanceRoot, + root: storageRoot, + cacheRoot: cacheRootValue, + fs, + path, + crypto, + container, + spawner, + }); + const makeStorage = (instanceId: string, instanceRoot: string) => + makeStorageWithCache(instanceId, instanceRoot, cacheRoot); + let sourceVolume: string | undefined; + const ownerScope = yield* Scope.Scope; + yield* Scope.addFinalizer( + ownerScope, + Effect.gen(function* () { + if (sourceVolume !== undefined) yield* engine(["volume", "rm", sourceVolume]); + yield* fs.remove(cacheRoot, { recursive: true, force: true }); + }).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.catchCause((cause) => Effect.die(cause)), + ), + ); + yield* Effect.scoped( + Effect.gen(function* () { + const source = yield* makeStorage("source", sourceRoot); + yield* source.prepare("17"); + const mount = yield* source.mount("17"); + if (mount.type !== "volume" || mount.volumeSubpath === undefined) + return yield* new DockerTestError({ message: "Docker volume backend unavailable" }); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), + ); + sourceVolume = marker.volume; + if (sourceVolume === undefined) + return yield* new DockerTestError({ message: "Docker storage volume missing" }); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceVolume},dst=/data,volume-subpath=${mount.volumeSubpath}`, + helperImage, + "/bin/sh", + "-c", + "printf 17 > /data/PG_VERSION; printf reopened > /data/fixture", + ]); + yield* source.markInitialized("17"); + yield* fs.writeFileString( + path.join(sourceRoot, ".supabase-database-ready.json"), + '{"version":"17","runtime":"docker","profile":"supabase"}', + ); + yield* source.saveSnapshot("17", "reopened"); + }), + ); + yield* Effect.scoped( + Effect.gen(function* () { + const changedCacheRoot = path.join(root, "changed-cache"); + yield* fs.makeDirectory(changedCacheRoot, { recursive: true }); + const changedCache = yield* makeStorageWithCache( + "source", + sourceRoot, + changedCacheRoot, + ); + yield* changedCache.prepare("17"); + yield* changedCache.removeData("17"); + expect(yield* changedCache.restoreSnapshot("17", "reopened")).toBe(false); + const source = yield* makeStorage("source", sourceRoot); + yield* source.prepare("17"); + yield* source.removeData("17"); + expect(yield* source.restoreSnapshot("17", "reopened")).toBe(true); + yield* source.saveSnapshot("17", "reopened-again"); + yield* source.destroyData("17"); + const target = yield* makeStorage("target", targetRoot); + yield* target.prepare("17"); + expect(yield* target.restoreSnapshot("17", "reopened-again")).toBe(true); + const targetMount = yield* target.mount("17"); + if (targetMount.type !== "volume" || targetMount.volumeSubpath === undefined) + return yield* new DockerTestError({ message: "Docker target volume unavailable" }); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceVolume ?? ""},dst=/data,volume-subpath=${targetMount.volumeSubpath}`, + helperImage, + "/bin/sh", + "-c", + 'test "$(cat /data/fixture)" = reopened', + ]); + yield* target.destroyData("17"); + }), + ); }), ).pipe(Effect.provide(NodeServices.layer)), - ); + ); + }, +); +describe("Host database storage", { timeout: 120_000 }, () => { it.live("handles root-owned host data through helper operations", () => Effect.scoped( Effect.gen(function* () { @@ -1444,14 +1594,14 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* fs.writeFileString(path.join(dataRoot, "fixture"), "adopted"); yield* fs.writeFileString( path.join(instanceRoot, ".supabase-database-ready.json"), - '{"version":"17","runtime":"docker","profile":"supabase"}', + `{"version":"17","runtime":"${testEngine}","profile":"supabase"}`, ); const container = yield* makeContainerRuntime({ - target: dockerTarget, + target: containerTarget, root, }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - yield* docker([ + yield* engine([ "run", "--rm", "--mount", @@ -1462,8 +1612,8 @@ describe("Docker database storage", { timeout: 120_000 }, () => { "chown -R 100:101 /instance/data; chmod 700 /instance/data", ]); const storage = yield* makeDockerDatabaseStorage({ - runtime: "docker", - target: dockerTarget, + runtime: testEngine, + target: containerTarget, stackId: "storage-host-test", instanceId: "adopted", instanceRoot, @@ -1489,7 +1639,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { ); expect(marker.backend).toBe("host"); expect(marker.initialized).toBe(true); - const expectedOwnership = yield* docker([ + const expectedOwnership = yield* engine([ "run", "--rm", "--mount", @@ -1523,7 +1673,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* storage.removeData("17"); expect(yield* storage.restoreSnapshot("17", "adopted")).toBe(true); expect( - yield* docker([ + yield* engine([ "run", "--rm", "--mount", @@ -1547,130 +1697,4 @@ describe("Docker database storage", { timeout: 120_000 }, () => { }), ).pipe(Effect.provide(NodeServices.layer)), ); - - it.live("reopens managed storage and preserves snapshots after source destruction", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const helperImage = yield* postgresImage("17"); - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-reopen-" }); - const stateRoot = path.join(root, "state"); - const storageRoot = path.join(stateRoot, "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const sourceRoot = path.join(storageRoot, "source"); - const targetRoot = path.join(storageRoot, "target"); - yield* fs.makeDirectory(sourceRoot, { recursive: true }); - yield* fs.makeDirectory(targetRoot, { recursive: true }); - yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ - target: dockerTarget, - root, - }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const makeStorageWithCache = ( - instanceId: string, - instanceRoot: string, - cacheRootValue: string, - ) => - makeDockerDatabaseStorage({ - runtime: "docker", - target: dockerTarget, - stackId: "storage-reopen-test", - instanceId, - instanceRoot, - root: storageRoot, - cacheRoot: cacheRootValue, - fs, - path, - crypto, - container, - spawner, - }); - const makeStorage = (instanceId: string, instanceRoot: string) => - makeStorageWithCache(instanceId, instanceRoot, cacheRoot); - let sourceVolume: string | undefined; - const ownerScope = yield* Scope.Scope; - yield* Scope.addFinalizer( - ownerScope, - Effect.gen(function* () { - if (sourceVolume !== undefined) yield* docker(["volume", "rm", sourceVolume]); - yield* fs.remove(cacheRoot, { recursive: true, force: true }); - }).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - Effect.catchCause((cause) => Effect.die(cause)), - ), - ); - yield* Effect.scoped( - Effect.gen(function* () { - const source = yield* makeStorage("source", sourceRoot); - yield* source.prepare("17"); - const mount = yield* source.mount("17"); - if (mount.type !== "volume" || mount.volumeSubpath === undefined) - return yield* new DockerTestError({ message: "Docker volume backend unavailable" }); - const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), - ); - sourceVolume = marker.volume; - if (sourceVolume === undefined) - return yield* new DockerTestError({ message: "Docker storage volume missing" }); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceVolume},dst=/data,volume-subpath=${mount.volumeSubpath}`, - helperImage, - "/bin/sh", - "-c", - "printf 17 > /data/PG_VERSION; printf reopened > /data/fixture", - ]); - yield* source.markInitialized("17"); - yield* fs.writeFileString( - path.join(sourceRoot, ".supabase-database-ready.json"), - '{"version":"17","runtime":"docker","profile":"supabase"}', - ); - yield* source.saveSnapshot("17", "reopened"); - }), - ); - yield* Effect.scoped( - Effect.gen(function* () { - const changedCacheRoot = path.join(root, "changed-cache"); - yield* fs.makeDirectory(changedCacheRoot, { recursive: true }); - const changedCache = yield* makeStorageWithCache( - "source", - sourceRoot, - changedCacheRoot, - ); - yield* changedCache.prepare("17"); - yield* changedCache.removeData("17"); - expect(yield* changedCache.restoreSnapshot("17", "reopened")).toBe(false); - const source = yield* makeStorage("source", sourceRoot); - yield* source.prepare("17"); - yield* source.removeData("17"); - expect(yield* source.restoreSnapshot("17", "reopened")).toBe(true); - yield* source.saveSnapshot("17", "reopened-again"); - yield* source.destroyData("17"); - const target = yield* makeStorage("target", targetRoot); - yield* target.prepare("17"); - expect(yield* target.restoreSnapshot("17", "reopened-again")).toBe(true); - const targetMount = yield* target.mount("17"); - if (targetMount.type !== "volume" || targetMount.volumeSubpath === undefined) - return yield* new DockerTestError({ message: "Docker target volume unavailable" }); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceVolume ?? ""},dst=/data,volume-subpath=${targetMount.volumeSubpath}`, - helperImage, - "/bin/sh", - "-c", - 'test "$(cat /data/fixture)" = reopened', - ]); - yield* target.destroyData("17"); - }), - ); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); }); diff --git a/packages/stack/src/whole-stack.podman.idle-parallel.e2e.test.ts b/packages/stack/src/whole-stack.podman.idle-parallel.e2e.test.ts new file mode 100644 index 0000000000..bd6d8a968e --- /dev/null +++ b/packages/stack/src/whole-stack.podman.idle-parallel.e2e.test.ts @@ -0,0 +1,15 @@ +import { it } from "@effect/vitest"; +import { parallel } from "../tests/whole-stack/scenarios.ts"; +import { functionsIdleWake, idleWake, restartIdleWake } from "../tests/whole-stack/idle.ts"; +import { run } from "../tests/whole-stack/fixture.ts"; + +it.live("Podman: idle and wake", () => run(idleWake("podman")), { timeout: 10 * 60_000 }); +it.live("Podman: restart preserves wake and idles again", () => run(restartIdleWake("podman")), { + timeout: 10 * 60_000, +}); +it.live("Podman: functions idles and wakes", () => run(functionsIdleWake("podman")), { + timeout: 10 * 60_000, +}); +it.live("Podman: parallel stack isolation", () => run(parallel("podman")), { + timeout: 30 * 60_000, +}); diff --git a/packages/stack/src/whole-stack.podman.lifecycle.e2e.test.ts b/packages/stack/src/whole-stack.podman.lifecycle.e2e.test.ts new file mode 100644 index 0000000000..b94bbc7aea --- /dev/null +++ b/packages/stack/src/whole-stack.podman.lifecycle.e2e.test.ts @@ -0,0 +1,17 @@ +import { it } from "@effect/vitest"; +import { allEager, defaultLifecycle, writeConfinement } from "../tests/whole-stack/scenarios.ts"; +import { run } from "../tests/whole-stack/fixture.ts"; + +it.live( + "Podman: default database eager lifecycle and reopen", + () => run(defaultLifecycle("podman")), + { + timeout: 15 * 60_000, + }, +); +it.live("Podman: all services eager", () => run(allEager("podman")), { timeout: 15 * 60_000 }); +it.live( + "Podman: writes stay confined to the state root and artifact cache", + () => run(writeConfinement("podman")), + { timeout: 15 * 60_000 }, +); diff --git a/packages/stack/tests/docker-cleanup.ts b/packages/stack/tests/docker-cleanup.ts index cf64d569b6..a779e3b21c 100644 --- a/packages/stack/tests/docker-cleanup.ts +++ b/packages/stack/tests/docker-cleanup.ts @@ -1,5 +1,6 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { Effect, Stream } from "effect"; +import { testEngine } from "./test-engine.ts"; export const cleanupDockerRoot = Effect.fn("DockerTest.cleanupRoot")((root: string) => Effect.scoped( @@ -7,7 +8,7 @@ export const cleanupDockerRoot = Effect.fn("DockerTest.cleanupRoot")((root: stri const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( ChildProcess.make( - "docker", + testEngine, [ "run", "--rm", @@ -15,7 +16,7 @@ export const cleanupDockerRoot = Effect.fn("DockerTest.cleanupRoot")((root: stri "0", "--mount", `type=bind,src=${root},dst=/mnt`, - "busybox:1.36", + "docker.io/library/busybox:1.36", "find", "/mnt", "-mindepth", @@ -30,7 +31,7 @@ export const cleanupDockerRoot = Effect.fn("DockerTest.cleanupRoot")((root: stri { concurrency: "unbounded" }, ); if (Number(code) !== 0) - return yield* Effect.die(`Docker test cleanup exited with ${code}: ${output}`); + return yield* Effect.die(`${testEngine} test cleanup exited with ${code}: ${output}`); }), ).pipe(Effect.catchCause(Effect.die)), ); diff --git a/packages/stack/tests/docker-fixture.integration.test.ts b/packages/stack/tests/docker-fixture.integration.test.ts index cecbd1d666..444ecdd2d6 100644 --- a/packages/stack/tests/docker-fixture.integration.test.ts +++ b/packages/stack/tests/docker-fixture.integration.test.ts @@ -6,7 +6,7 @@ import { Context, Effect, FileSystem, Layer, Redacted, Schema, Stream } from "ef import { makeStandaloneService } from "./standalone-service.ts"; import { makeDatabase, type BackendEndpoint } from "../src/services/Database.ts"; import { makeDockerDatabaseRoot } from "./docker-fixture.ts"; -import { dockerEngineTarget } from "./engine-target.ts"; +import { engineTarget, testEngine } from "./engine-target.ts"; const marker = Schema.Struct({ backend: Schema.Literal("docker"), volume: Schema.String }); const password = Redacted.make("supabase-test-password"); @@ -54,90 +54,98 @@ const volumeExists = Effect.fn("DockerFixture.volumeExists")((volume: string) => ), ); -describe("Docker database fixture isolation", { timeout: 180_000 }, () => { - it.live("isolates equal identities across separate owned volumes", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const firstRoot = yield* makeDockerDatabaseRoot( - "docker-fixture-first-", - "docker-fixture-isolation", - ); - const secondRoot = yield* makeDockerDatabaseRoot( - "docker-fixture-second-", - "docker-fixture-isolation", - ); - const make = (root: string) => - makeDatabase({ - stackId: "docker-fixture-isolation", - instanceId: "database", - root, - cacheRoot: `${root}/cache`, - runtime: "docker", - engineTarget: dockerEngineTarget, +describe.runIf(testEngine === "docker")( + "Docker database fixture isolation", + { timeout: 180_000 }, + () => { + it.live("isolates equal identities across separate owned volumes", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const firstRoot = yield* makeDockerDatabaseRoot( + "docker-fixture-first-", + "docker-fixture-isolation", + ); + const secondRoot = yield* makeDockerDatabaseRoot( + "docker-fixture-second-", + "docker-fixture-isolation", + ); + const make = (root: string) => + makeDatabase({ + stackId: "docker-fixture-isolation", + instanceId: "database", + root, + cacheRoot: `${root}/cache`, + runtime: "docker", + engineTarget, + }); + const [first, second] = yield* Effect.all([make(firstRoot), make(secondRoot)]); + const [firstService, secondService] = yield* Effect.all([ + makeStandaloneService(first.definition, { + id: "docker-fixture-first", + config: { + version: "17", + databasePassword: password, + jwtSecret: Redacted.make("fixture-secret-with-at-least-32-chars"), + jwtExpiry: 3600, + }, + }), + makeStandaloneService(second.definition, { + id: "docker-fixture-second", + config: { + version: "17", + databasePassword: password, + jwtSecret: Redacted.make("fixture-secret-with-at-least-32-chars"), + jwtExpiry: 3600, + }, + }), + ]); + yield* Effect.all([firstService.start, secondService.start], { + concurrency: "unbounded", }); - const [first, second] = yield* Effect.all([make(firstRoot), make(secondRoot)]); - const [firstService, secondService] = yield* Effect.all([ - makeStandaloneService(first.definition, { - id: "docker-fixture-first", - config: { - version: "17", - databasePassword: password, - jwtSecret: Redacted.make("fixture-secret-with-at-least-32-chars"), - jwtExpiry: 3600, - }, - }), - makeStandaloneService(second.definition, { - id: "docker-fixture-second", - config: { - version: "17", - databasePassword: password, - jwtSecret: Redacted.make("fixture-secret-with-at-least-32-chars"), - jwtExpiry: 3600, - }, - }), - ]); - yield* Effect.all([firstService.start, secondService.start], { concurrency: "unbounded" }); - yield* Effect.all([firstService.ready, secondService.ready], { concurrency: "unbounded" }); - const [firstEndpoint, secondEndpoint] = yield* Effect.all([ - first.endpoint, - second.endpoint, - ]); - yield* query(firstEndpoint, "CREATE TABLE fixture_isolation (value text)"); - yield* query(firstEndpoint, "INSERT INTO fixture_isolation VALUES ('first')"); - yield* query(secondEndpoint, "CREATE TABLE fixture_isolation (value text)"); - yield* query(secondEndpoint, "INSERT INTO fixture_isolation VALUES ('second')"); - expect(yield* query(firstEndpoint, "SELECT value FROM fixture_isolation")).toEqual([ - { value: "first" }, - ]); - expect(yield* query(secondEndpoint, "SELECT value FROM fixture_isolation")).toEqual([ - { value: "second" }, - ]); - yield* firstService.stop; - expect(yield* query(secondEndpoint, "SELECT value FROM fixture_isolation")).toEqual([ - { value: "second" }, - ]); - const markers = yield* Effect.all( - [firstRoot, secondRoot].map((root) => - fs - .readFileString(`${root}/database/.supabase-database-storage.json`) - .pipe(Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(marker)))), + yield* Effect.all([firstService.ready, secondService.ready], { + concurrency: "unbounded", + }); + const [firstEndpoint, secondEndpoint] = yield* Effect.all([ + first.endpoint, + second.endpoint, + ]); + yield* query(firstEndpoint, "CREATE TABLE fixture_isolation (value text)"); + yield* query(firstEndpoint, "INSERT INTO fixture_isolation VALUES ('first')"); + yield* query(secondEndpoint, "CREATE TABLE fixture_isolation (value text)"); + yield* query(secondEndpoint, "INSERT INTO fixture_isolation VALUES ('second')"); + expect(yield* query(firstEndpoint, "SELECT value FROM fixture_isolation")).toEqual([ + { value: "first" }, + ]); + expect(yield* query(secondEndpoint, "SELECT value FROM fixture_isolation")).toEqual([ + { value: "second" }, + ]); + yield* firstService.stop; + expect(yield* query(secondEndpoint, "SELECT value FROM fixture_isolation")).toEqual([ + { value: "second" }, + ]); + const markers = yield* Effect.all( + [firstRoot, secondRoot].map((root) => + fs + .readFileString(`${root}/database/.supabase-database-storage.json`) + .pipe(Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(marker)))), + ), + ); + yield* secondService.destroy; + yield* firstService.destroy; + return markers.map((entry) => entry.volume); + }), + ).pipe( + Effect.flatMap((volumes) => + Effect.forEach(volumes, (volume) => + volumeExists(volume).pipe(Effect.map((exists) => ({ volume, exists }))), ), - ); - yield* secondService.destroy; - yield* firstService.destroy; - return markers.map((entry) => entry.volume); - }), - ).pipe( - Effect.flatMap((volumes) => - Effect.forEach(volumes, (volume) => - volumeExists(volume).pipe(Effect.map((exists) => ({ volume, exists }))), ), + Effect.tap((results) => + Effect.sync(() => results.forEach(({ exists }) => expect(exists).toBe(true))), + ), + Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp)), ), - Effect.tap((results) => - Effect.sync(() => results.forEach(({ exists }) => expect(exists).toBe(true))), - ), - Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp)), - ), - ); -}); + ); + }, +); diff --git a/packages/stack/tests/docker-fixture.ts b/packages/stack/tests/docker-fixture.ts index a900828687..b8ada315ca 100644 --- a/packages/stack/tests/docker-fixture.ts +++ b/packages/stack/tests/docker-fixture.ts @@ -1,14 +1,18 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { Effect, FileSystem, Stream } from "effect"; import { cleanupDockerRoot } from "./docker-cleanup.ts"; +import { testEngine } from "./test-engine.ts"; -/** Runs a Docker CLI command and returns its combined output and exit code. */ -export const runDocker = Effect.fn("DockerTest.runDocker")((args: ReadonlyArray) => +/** + * Runs a command of the selected test engine's CLI and returns its combined output and exit code. + * It does not pin the engine target, so it works before an engine is known to be reachable. + */ +export const runEngine = Effect.fn("DockerTest.runEngine")((args: ReadonlyArray) => Effect.scoped( Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", args, { stdout: "pipe", stderr: "pipe" }), + ChildProcess.make(testEngine, args, { stdout: "pipe", stderr: "pipe" }), ); const [stdout, stderr, code] = yield* Effect.all( [ diff --git a/packages/stack/tests/docker-volume-run.integration.test.ts b/packages/stack/tests/docker-volume-run.integration.test.ts index 6233923a8f..3cb07456d5 100644 --- a/packages/stack/tests/docker-volume-run.integration.test.ts +++ b/packages/stack/tests/docker-volume-run.integration.test.ts @@ -2,7 +2,7 @@ import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Crypto, DateTime, Effect, FileSystem, Path } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { runDocker } from "./docker-fixture.ts"; +import { runEngine } from "./docker-fixture.ts"; import { encodeRunMarker, markerDirectory, @@ -12,12 +12,12 @@ import { } from "./docker-volume-run.ts"; const volumeExists = Effect.fn("DockerVolumeRunTest.volumeExists")((name: string) => - runDocker(["volume", "inspect", name]).pipe(Effect.map((result) => result.code === 0)), + runEngine(["volume", "inspect", name]).pipe(Effect.map((result) => result.code === 0)), ); const createVolume = Effect.fn("DockerVolumeRunTest.createVolume")( (name: string, labels: ReadonlyArray) => - runDocker(["volume", "create", ...labels.flatMap((label) => ["--label", label]), name]).pipe( + runEngine(["volume", "create", ...labels.flatMap((label) => ["--label", label]), name]).pipe( Effect.flatMap((result) => result.code === 0 ? Effect.void @@ -27,7 +27,7 @@ const createVolume = Effect.fn("DockerVolumeRunTest.createVolume")( ); const removeVolume = Effect.fn("DockerVolumeRunTest.removeVolume")((name: string) => - runDocker(["volume", "rm", name]).pipe( + runEngine(["volume", "rm", name]).pipe( Effect.flatMap((result) => result.code === 0 || /no such volume/iu.test(result.output) ? Effect.void @@ -38,17 +38,17 @@ const removeVolume = Effect.fn("DockerVolumeRunTest.removeVolume")((name: string ); const containerExists = Effect.fn("DockerVolumeRunTest.containerExists")((name: string) => - runDocker(["inspect", name]).pipe(Effect.map((result) => result.code === 0)), + runEngine(["inspect", name]).pipe(Effect.map((result) => result.code === 0)), ); const createContainer = Effect.fn("DockerVolumeRunTest.createContainer")( (name: string, labels: ReadonlyArray) => - runDocker([ + runEngine([ "create", "--name", name, ...labels.flatMap((label) => ["--label", label]), - "busybox:1.36", + "docker.io/library/busybox:1.36", "true", ]).pipe( Effect.flatMap((result) => @@ -60,7 +60,7 @@ const createContainer = Effect.fn("DockerVolumeRunTest.createContainer")( ); const removeContainer = Effect.fn("DockerVolumeRunTest.removeContainer")((name: string) => - runDocker(["rm", "--force", "--volumes", name]).pipe( + runEngine(["rm", "--force", "--volumes", name]).pipe( Effect.flatMap((result) => result.code === 0 || /no such container/iu.test(result.output) ? Effect.void diff --git a/packages/stack/tests/docker-volume-run.ts b/packages/stack/tests/docker-volume-run.ts index 14d469ae63..398c555c39 100644 --- a/packages/stack/tests/docker-volume-run.ts +++ b/packages/stack/tests/docker-volume-run.ts @@ -6,7 +6,7 @@ import { testRunEnvVar as stackTestRunEnvVar, testRunLabelKey, } from "../src/internal/test-run-label.ts"; -import { runDocker } from "./docker-fixture.ts"; +import { runEngine } from "./docker-fixture.ts"; /** * Docker test state roots must be private to a run (see `makeDockerDatabaseRoot` and @@ -44,7 +44,7 @@ const isDead = (pid: number): boolean => { } }; -const dockerUnavailable = runDocker(["info", "--format", "{{.ID}}"]).pipe( +const dockerUnavailable = runEngine(["version"]).pipe( Effect.map((result) => result.code !== 0), Effect.catchCause(() => Effect.succeed(true)), ); @@ -62,7 +62,7 @@ export const removeTestRunVolumes = Effect.fn("DockerVolumeRun.removeTestRunVolu ); return; } - const listed = yield* runDocker([ + const listed = yield* runEngine([ "volume", "ls", "-q", @@ -79,7 +79,7 @@ export const removeTestRunVolumes = Effect.fn("DockerVolumeRun.removeTestRunVolu .filter((line) => line.length > 0); if (volumes.length === 0) return; const removals = yield* Effect.forEach(volumes, (volume) => - runDocker(["volume", "rm", volume]).pipe(Effect.map((result) => ({ volume, result }))), + runEngine(["volume", "rm", volume]).pipe(Effect.map((result) => ({ volume, result }))), ); const failed = removals.filter(({ result }) => result.code !== 0); if (failed.length > 0) @@ -97,7 +97,7 @@ const benign = (output: string) => /no such (?:container|volume)/iu.test(output) /** Removes a dead run's labelled containers; `Option.none` on success, a message otherwise. */ const removeDeadRunContainers = Effect.fn("DockerVolumeRun.removeDeadRunContainers")((id: string) => Effect.gen(function* () { - const listed = yield* runDocker([ + const listed = yield* runEngine([ "ps", "--all", "--quiet", @@ -111,7 +111,7 @@ const removeDeadRunContainers = Effect.fn("DockerVolumeRun.removeDeadRunContaine .map((line) => line.trim()) .filter((line) => line.length > 0); if (ids.length === 0) return Option.none(); - const removed = yield* runDocker(["rm", "--force", "--volumes", ...ids]); + const removed = yield* runEngine(["rm", "--force", "--volumes", ...ids]); return removed.code === 0 || benign(removed.output) ? Option.none() : Option.some(`docker rm failed: ${removed.output.trim()}`); @@ -121,7 +121,7 @@ const removeDeadRunContainers = Effect.fn("DockerVolumeRun.removeDeadRunContaine /** Removes a dead run's labelled, stack-managed volumes; `Option.none` on success. */ const removeDeadRunVolumes = Effect.fn("DockerVolumeRun.removeDeadRunVolumes")((id: string) => Effect.gen(function* () { - const listed = yield* runDocker([ + const listed = yield* runEngine([ "volume", "ls", "-q", @@ -137,7 +137,7 @@ const removeDeadRunVolumes = Effect.fn("DockerVolumeRun.removeDeadRunVolumes")(( .filter((line) => line.length > 0); if (volumes.length === 0) return Option.none(); const removals = yield* Effect.forEach(volumes, (volume) => - runDocker(["volume", "rm", volume]).pipe(Effect.map((result) => ({ volume, result }))), + runEngine(["volume", "rm", volume]).pipe(Effect.map((result) => ({ volume, result }))), ); const failed = removals.filter(({ result }) => result.code !== 0 && !benign(result.output)); return failed.length === 0 diff --git a/packages/stack/tests/engine-events.ts b/packages/stack/tests/engine-events.ts new file mode 100644 index 0000000000..b435e92aaa --- /dev/null +++ b/packages/stack/tests/engine-events.ts @@ -0,0 +1,110 @@ +import { Data, Effect, Option, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { engineTarget, testEngine } from "./engine-target.ts"; + +class EngineEventsError extends Data.TaggedError("EngineEventsError")<{ + readonly message: string; +}> {} + +const eventLines = Effect.fn("EngineEvents.eventLines")((args: ReadonlyArray) => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const events = yield* spawner.spawn( + ChildProcess.make(testEngine, [...engineTarget.argv, "events", ...args], { + stdin: "ignore", + stdout: "pipe", + stderr: "ignore", + }), + ); + return events.stdout.pipe(Stream.decodeText, Stream.splitLines); + }), +); + +/** + * Completes when the engine reports the container removed, including a removal that happened + * after `sinceSeconds` and before the stream attached. Docker names that event `destroy`; Podman + * names it `remove`. + */ +export const awaitContainerRemoved = (id: string, sinceSeconds: number) => + Effect.scoped( + Effect.gen(function* () { + const lines = yield* eventLines( + testEngine === "docker" + ? [ + "--since", + String(sinceSeconds), + "--filter", + `container=${id}`, + "--filter", + "event=destroy", + "--format", + "{{.Actor.ID}}", + ] + : [ + "--since", + String(sinceSeconds), + "--filter", + `container=${id}`, + "--filter", + "event=remove", + "--format", + "{{.ID}}", + ], + ); + const removed = yield* lines.pipe( + Stream.filter((line) => line.trim() === id), + Stream.runHead, + ); + if (Option.isNone(removed)) + return yield* new EngineEventsError({ message: `${testEngine} event stream ended` }); + }), + ); + +/** + * What the engine reports for a container stopped with SIGINT that exits cleanly. Docker's stop + * sends the signal through the daemon, so it reports `kill` with the signal before `die`; Podman's + * stop signals the container directly and reports only `died` with the exit code. + */ +export const cleanStopEvents: ReadonlyArray = + testEngine === "docker" ? ["kill 2", "die 0"] : ["died 0"]; + +/** + * Subscribes to the container's stop events from `since` (replaying any logged before the stream + * attaches) and resolves, once its terminal event arrives, to every line observed. Subscribing + * before triggering the stop avoids Docker Desktop's intermittently empty bounded replays. + */ +export const observeContainerStop = Effect.fn("EngineEvents.observeContainerStop")(function* ( + container: string, + since: string, +) { + const lines = yield* eventLines( + testEngine === "docker" + ? [ + "--since", + since, + "--filter", + `container=${container}`, + "--filter", + "event=kill", + "--filter", + "event=die", + "--format", + '{{.Action}} {{index .Actor.Attributes "signal"}}{{index .Actor.Attributes "exitCode"}}', + ] + : [ + "--since", + since, + "--filter", + `container=${container}`, + "--filter", + "event=died", + "--format", + "{{.Status}} {{.ContainerExitCode}}", + ], + ); + return yield* lines.pipe( + Stream.takeUntil((line) => line.startsWith(testEngine === "docker" ? "die " : "died ")), + Stream.runCollect, + Effect.forkScoped, + ); +}); diff --git a/packages/stack/tests/engine-target.ts b/packages/stack/tests/engine-target.ts index 3e5fb0eb84..7bab3eb8e8 100644 --- a/packages/stack/tests/engine-target.ts +++ b/packages/stack/tests/engine-target.ts @@ -2,12 +2,15 @@ import { NodeServices } from "@effect/platform-node"; import { Effect } from "effect"; import { ChildProcessSpawner } from "effect/unstable/process"; import { resolveEngineTarget, type EngineTarget } from "../src/runtime/Container.ts"; +import { testEngine } from "./test-engine.ts"; + +export { testEngine }; /** - * Resolves the real docker engine target once per test file, for `it.live` tests that exercise a - * service catalog or a database against the real daemon rather than a fake one. + * Resolves the selected engine target once per test file, for `it.live` tests that exercise a + * service catalog or a database against the real engine rather than a fake one. */ -export const dockerEngineTarget: EngineTarget = await Effect.gen(function* () { +export const engineTarget: EngineTarget = await Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - return yield* resolveEngineTarget(spawner); + return yield* resolveEngineTarget(spawner, testEngine); }).pipe(Effect.provide(NodeServices.layer), Effect.runPromise); diff --git a/packages/stack/tests/gated-docker-stop-fixture.ts b/packages/stack/tests/gated-docker-stop-fixture.ts index ae0be6da0a..b3796d7ea8 100644 --- a/packages/stack/tests/gated-docker-stop-fixture.ts +++ b/packages/stack/tests/gated-docker-stop-fixture.ts @@ -30,7 +30,7 @@ const writeLine = (value: unknown) => /** * A real owner entrypoint, identical to `internal/host-process.ts`, except it puts - * `/bin` first on PATH so a test can interpose on the `docker` commands the owner runs. + * `/bin` first on PATH so a test can interpose on the container engine commands the owner runs. */ const [stateRoot, cacheRoot, stackId, gateDir, ...rest] = process.argv.slice(2); diff --git a/packages/stack/tests/test-engine.ts b/packages/stack/tests/test-engine.ts new file mode 100644 index 0000000000..fcfac1159b --- /dev/null +++ b/packages/stack/tests/test-engine.ts @@ -0,0 +1,12 @@ +import { Config, Effect } from "effect"; +import type { ContainerEngine } from "../src/runtime/Container.ts"; + +/** + * The container engine real-engine tests run against: `docker` unless `SUPABASE_STACK_TEST_ENGINE` + * selects `podman`. Resolving it never contacts the engine. + */ +export const testEngine: ContainerEngine = Effect.runSync( + Config.literals(["docker", "podman"], "SUPABASE_STACK_TEST_ENGINE").pipe( + Config.withDefault("docker"), + ), +); diff --git a/packages/stack/tests/whole-stack/confinement.ts b/packages/stack/tests/whole-stack/confinement.ts index dc882349e1..03a61a0d7b 100644 --- a/packages/stack/tests/whole-stack/confinement.ts +++ b/packages/stack/tests/whole-stack/confinement.ts @@ -1,5 +1,6 @@ import { expect } from "@effect/vitest"; import { Effect, FileSystem, Option, Path } from "effect"; +import type { PlatformError } from "effect/PlatformError"; interface TreeEntry { readonly isFile: boolean; @@ -9,36 +10,60 @@ interface TreeEntry { export type Tree = ReadonlyMap; -/** Snapshots every path under `root`, or an empty tree when `root` does not exist yet. */ -export const snapshotTree = Effect.fn("Confinement.snapshotTree")(function* (root: string) { +/** + * Snapshots every path under `root`, or an empty tree when `root` does not exist yet. A directory + * listed in `opaque` is recorded but never entered, for data a container engine owns under a + * mapped uid that the host user cannot traverse. + */ +export const snapshotTree = Effect.fn("Confinement.snapshotTree")(function* ( + root: string, + opaque: ReadonlyArray = [], +) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; if (!(yield* fs.exists(root))) return new Map(); - const entries = yield* fs.readDirectory(root, { recursive: true }); - const stats = yield* Effect.forEach( - entries, - (entry) => { - const full = path.join(root, entry); - return fs.stat(full).pipe( - Effect.map((info): readonly [string, TreeEntry] => [ - full, - { - isFile: info.type === "File", - size: info.size, - mtimeMs: Option.match(info.mtime, { - onNone: () => 0, - onSome: (date) => date.getTime(), - }), - }, - ]), - // A path can disappear between listing and stat (e.g. a service's own temp file); treat it - // as absent rather than failing the whole snapshot. - Effect.orElseSucceed(() => [full, { isFile: false, size: 0n, mtimeMs: 0 }] as const), + const entries = new Map(); + const visit = (directory: string): Effect.Effect => + Effect.gen(function* () { + const names = yield* fs.readDirectory(directory); + yield* Effect.forEach( + names, + (name) => + Effect.gen(function* () { + const full = path.join(directory, name); + const info = yield* fs.stat(full).pipe(Effect.option); + // A path can disappear between listing and stat (e.g. a service's own temp file); treat + // it as absent rather than failing the whole snapshot. + if (Option.isNone(info)) { + entries.set(full, { isFile: false, size: 0n, mtimeMs: 0 }); + return; + } + entries.set(full, { + isFile: info.value.type === "File", + size: info.value.size, + mtimeMs: Option.match(info.value.mtime, { + onNone: () => 0, + onSome: (date) => date.getTime(), + }), + }); + // A symlink is recorded but not followed, matching a recursive directory listing. + if ( + info.value.type === "Directory" && + !opaque.includes(full) && + (yield* fs.readLink(full).pipe(Effect.isFailure)) + ) + yield* visit(full); + }), + { concurrency: 8, discard: true }, ); - }, - { concurrency: 8 }, - ); - return new Map(stats); + }).pipe( + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.void, + ), + ); + yield* visit(root); + return entries as Tree; }); export interface ConfinementViolation { diff --git a/packages/stack/tests/whole-stack/fixture.ts b/packages/stack/tests/whole-stack/fixture.ts index 98e048cf87..cf7c258e9c 100644 --- a/packages/stack/tests/whole-stack/fixture.ts +++ b/packages/stack/tests/whole-stack/fixture.ts @@ -16,10 +16,12 @@ import { postgres } from "../../src/Commands.ts"; import { homedir, tmpdir } from "node:os"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { create, type Stack } from "../../src/effect.ts"; +import type { ContainerEngine } from "../../src/runtime/Container.ts"; import type { Observation } from "../../src/Rpc.ts"; import { vectorAnalyticsConfig } from "./analytics.ts"; import { cleanupDockerRoot } from "../docker-cleanup.ts"; import { destroyTestStack } from "../stack-cleanup.ts"; +import { testEngine } from "../test-engine.ts"; type AnyService = Effect.Success[number]; @@ -39,7 +41,7 @@ export const serviceNames = [ "pooler", ] as const; -export type Runtime = "native" | "docker"; +export type Runtime = "native" | ContainerEngine; export type WholeStack = Readonly<{ readonly stack: Stack; readonly services: ReadonlyArray; @@ -112,6 +114,12 @@ const ownerLogTail = Effect.fn("WholeStack.ownerLogTail")( export const wholeStack = Effect.fn("WholeStack.fixture")((runtime: Runtime) => Effect.gen(function* () { + // The test helpers observe and clean up through the selected engine, so a container runtime + // must be that engine. + if (runtime !== "native" && runtime !== testEngine) + return yield* Effect.die( + `Whole-stack ${runtime} scenarios need SUPABASE_STACK_TEST_ENGINE=${runtime}`, + ); const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: `stack-whole-${runtime}-`, @@ -144,7 +152,7 @@ export const wholeStack = Effect.fn("WholeStack.fixture")((runtime: Runtime) => Effect.gen(function* () { const current = yield* Ref.get(owner); const destroy = Option.isSome(current) ? destroyTestStack(current.value) : Effect.void; - yield* runtime === "docker" + yield* runtime !== "native" ? destroy.pipe(Effect.ensuring(cleanupDockerRoot(storageRoot))) : destroy; }), diff --git a/packages/stack/tests/whole-stack/scenarios.ts b/packages/stack/tests/whole-stack/scenarios.ts index 53b3f826eb..a829d08c20 100644 --- a/packages/stack/tests/whole-stack/scenarios.ts +++ b/packages/stack/tests/whole-stack/scenarios.ts @@ -1,7 +1,7 @@ import { expect } from "@effect/vitest"; -import { Cause, Effect, FileSystem, Path, Ref, Schema } from "effect"; +import { Cause, Config, Effect, FileSystem, Path, Ref, Schema } from "effect"; import { open } from "../../src/effect.ts"; -import { runDocker } from "../docker-fixture.ts"; +import { runEngine } from "../docker-fixture.ts"; import { allMembers, clearStackOwner, @@ -542,8 +542,8 @@ const resolveRealDockerHost = Effect.gen(function* () { // oxlint-disable-next-line effecttsgo/process-env-in-effect -- reads the real environment before it is sandboxed below. const configured = process.env.DOCKER_HOST; if (configured !== undefined && configured.length > 0) return configured; - const name = (yield* runDocker(["context", "show"])).output.trim(); - const host = (yield* runDocker([ + const name = (yield* runEngine(["context", "show"])).output.trim(); + const host = (yield* runEngine([ "context", "inspect", name, @@ -554,6 +554,23 @@ const resolveRealDockerHost = Effect.gen(function* () { return host; }); +/** + * Rootless Podman runs its engine in-process and keeps image and container storage under the XDG + * data and config roots, so the sandboxed owner must keep the real ones to find them. Like the + * docker CLI's client state, that engine state is outside this test's confinement contract. + */ +const resolveRealPodmanState = Effect.gen(function* () { + const home = yield* Config.string("HOME"); + return { + XDG_DATA_HOME: yield* Config.string("XDG_DATA_HOME").pipe( + Config.withDefault(`${home}/.local/share`), + ), + XDG_CONFIG_HOME: yield* Config.string("XDG_CONFIG_HOME").pipe( + Config.withDefault(`${home}/.config`), + ), + }; +}); + /** * Proves a running stack writes only inside its state root and artifact cache (plus the project * directory it was handed): points HOME, TMPDIR/TMP/TEMP and the XDG roots at fresh, empty @@ -582,6 +599,7 @@ export const writeConfinement = (runtime: Runtime) => // the sandboxed owner to a `DOCKER_HOST` pin instead of its real context is an accepted // tradeoff for that, not something this test tries to confine. const dockerHost = runtime === "docker" ? yield* resolveRealDockerHost : undefined; + const podmanState = runtime === "podman" ? yield* resolveRealPodmanState : undefined; yield* withSandboxEnvironment( { HOME: home, @@ -589,8 +607,8 @@ export const writeConfinement = (runtime: Runtime) => TMP: tmp, TEMP: tmp, XDG_CACHE_HOME: path.join(home, ".cache"), - XDG_CONFIG_HOME: path.join(home, ".config"), - XDG_DATA_HOME: path.join(home, ".local", "share"), + XDG_CONFIG_HOME: podmanState?.XDG_CONFIG_HOME ?? path.join(home, ".config"), + XDG_DATA_HOME: podmanState?.XDG_DATA_HOME ?? path.join(home, ".local", "share"), XDG_STATE_HOME: path.join(home, ".local", "state"), BUN_RUNTIME_TRANSPILER_CACHE_PATH: bunTranspilerCache, DOCKER_HOST: dockerHost ?? "", @@ -602,12 +620,27 @@ export const writeConfinement = (runtime: Runtime) => yield* clearIdleTimers(fixture); yield* fixture.stack.composition.start; yield* exerciseStack(fixture, "confinement"); + // Podman keeps database files on the host under a subordinate uid the host user cannot + // traverse, so that one directory is recorded but not entered; every other path under + // the roots is still diffed. + const opaque = + runtime === "podman" + ? [ + path.join( + fixture.locations.stateRoot, + fixture.stack.id, + "data", + service(fixture, "database").id, + "data", + ), + ] + : []; // Allowed roots the named entries below cover; add one line here to extend them. const allowedRoots = [fixture.root, fixture.locations.cacheRoot, bunTranspilerCache]; // While the stack is still running, after it has been exercised: a before/after-only // check misses anything a service writes and removes again before shutdown. const duringHome = yield* snapshotTree(home); - const duringTmp = yield* snapshotTree(tmp); + const duringTmp = yield* snapshotTree(tmp, opaque); assertConfinedTo(diffTrees(beforeHome, duringHome), allowedRoots); assertConfinedTo(diffTrees(beforeTmp, duringTmp), allowedRoots); yield* proveConfinementDetectsViolations(home, duringHome, allowedRoots); @@ -615,7 +648,7 @@ export const writeConfinement = (runtime: Runtime) => yield* fixture.stack.destroy; yield* clearStackOwner(fixture); assertConfinedTo(diffTrees(beforeHome, yield* snapshotTree(home)), allowedRoots); - assertConfinedTo(diffTrees(beforeTmp, yield* snapshotTree(tmp)), allowedRoots); + assertConfinedTo(diffTrees(beforeTmp, yield* snapshotTree(tmp, opaque)), allowedRoots); }), ); }), diff --git a/packages/stack/tests/whole-stack/workloads.ts b/packages/stack/tests/whole-stack/workloads.ts index 76546012b2..52f9e26080 100644 --- a/packages/stack/tests/whole-stack/workloads.ts +++ b/packages/stack/tests/whole-stack/workloads.ts @@ -37,7 +37,7 @@ const snapshotOutput = (runtime: Runtime, fixture: WholeStack, includeStopped: b } satisfies WorkloadSnapshot; } const marker = `com.supabase.stack=${fixture.stack.id}`; - const output = yield* commandOutput("docker", [ + const output = yield* commandOutput(runtime, [ "ps", ...(includeStopped ? ["--all"] : []), "--filter", @@ -78,8 +78,8 @@ export const assertWorkloadsGone = Effect.fn("WholeStack.assertWorkloadsGone")( Effect.gen(function* () { expect(snapshot.identities.length).toBeGreaterThan(0); const current = yield* snapshotOutput(runtime, fixture, includeStopped); - if (runtime === "docker") expect(current.identities).toEqual([]); - if (runtime === "docker" && includeStopped) expect(current.managedIdentities).toEqual([]); + if (runtime !== "native") expect(current.identities).toEqual([]); + if (runtime !== "native" && includeStopped) expect(current.managedIdentities).toEqual([]); for (const marker of snapshot.markers) expect(current.identities.some((identity) => identity.includes(marker))).toBe(false); for (const identity of snapshot.identities) From 13a2c7d7567a20e8eccc518892629b5b0a68af6f Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 13:37:22 +0200 Subject: [PATCH 091/142] test(stack): keep the registry traverse-bit test to POSIX hosts --- packages/stack/src/Ports.integration.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/stack/src/Ports.integration.test.ts b/packages/stack/src/Ports.integration.test.ts index b8e6b53324..09affd1598 100644 --- a/packages/stack/src/Ports.integration.test.ts +++ b/packages/stack/src/Ports.integration.test.ts @@ -460,7 +460,8 @@ it.live("reserveNativePort excludes a port a previous attempt lost from the next ).pipe(Effect.provide(NodeServices.layer)), ); -it.live( +// Windows has no POSIX mode bits and no stepped-down native database. +it.live.skipIf(process.platform === "win32")( "opening the port registry keeps the traverse bit a stepped-down database needs on its directory", () => Effect.scoped( From e997abc23b432f161d44633d166028924cb054b0 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 13:43:21 +0200 Subject: [PATCH 092/142] fix(stack): give the Functions container a HOME on engines that leave it unset --- packages/stack/src/services/Functions.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/stack/src/services/Functions.ts b/packages/stack/src/services/Functions.ts index 3707a4d9f6..7ab0f74698 100644 --- a/packages/stack/src/services/Functions.ts +++ b/packages/stack/src/services/Functions.ts @@ -133,6 +133,9 @@ export const makeSpec = ( creation.config.serviceRoleKey ?? (jwt === undefined ? undefined : yield* serviceJwt("service_role", jwt)); return { + // The image has no passwd entry: Docker then defaults HOME to /, Podman leaves it unset, + // and the runtime needs it to place its Deno cache. + ...(container ? { HOME: "/" } : {}), ...creation.config.env, ...(http === undefined ? {} : { EDGE_RUNTIME_PORT: String(http.port) }), SUPABASE_INTERNAL_FUNCTIONS_ROOT: root, From ad2169ddbb3cb94c05cd81a7b96570e9348edc71 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 13:43:22 +0200 Subject: [PATCH 093/142] test(stack): list Podman workloads through its own label template --- packages/stack/tests/whole-stack/workloads.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/stack/tests/whole-stack/workloads.ts b/packages/stack/tests/whole-stack/workloads.ts index 52f9e26080..abf144b567 100644 --- a/packages/stack/tests/whole-stack/workloads.ts +++ b/packages/stack/tests/whole-stack/workloads.ts @@ -37,13 +37,16 @@ const snapshotOutput = (runtime: Runtime, fixture: WholeStack, includeStopped: b } satisfies WorkloadSnapshot; } const marker = `com.supabase.stack=${fixture.stack.id}`; + // Docker's `ps` template reads a label through `.Label`; Podman's has no such method. + const label = (key: string) => + runtime === "docker" ? `{{.Label "${key}"}}` : `{{index .Labels "${key}"}}`; const output = yield* commandOutput(runtime, [ "ps", ...(includeStopped ? ["--all"] : []), "--filter", `label=${marker}`, "--format", - '{{.ID}}\t{{.Label "com.supabase.instance"}}\t{{.Label "com.supabase.stack-managed"}}', + `{{.ID}}\t${label("com.supabase.instance")}\t${label("com.supabase.stack-managed")}`, ]); const records = output .split("\n") From 055877d52910bf3bc2e476cd5e59090ed750b18b Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 13:43:23 +0200 Subject: [PATCH 094/142] test(stack): hand Podman's event stream an RFC 3339 start time --- packages/stack/src/services/Database.integration.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/stack/src/services/Database.integration.test.ts b/packages/stack/src/services/Database.integration.test.ts index bcacc1240a..cc1117614d 100644 --- a/packages/stack/src/services/Database.integration.test.ts +++ b/packages/stack/src/services/Database.integration.test.ts @@ -605,11 +605,14 @@ describe("database component", { timeout: 180_000 }, () => { ...engineTarget.argv, "inspect", "--format", - "{{.State.StartedAt}}", + "{{json .State.StartedAt}}", container, ]); - const observed = yield* observeContainerStop(container, startedAt.output.trim()); + const observed = yield* observeContainerStop( + container, + startedAt.output.trim().replaceAll('"', ""), + ); yield* Effect.scoped( Effect.gen(function* () { From 162832bcbe64283f136fae2b0437c6626ff00000 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 13:49:07 +0200 Subject: [PATCH 095/142] test(stack): treat rpm's package database as Podman's own state in the confinement check --- packages/stack/tests/whole-stack/scenarios.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/stack/tests/whole-stack/scenarios.ts b/packages/stack/tests/whole-stack/scenarios.ts index a829d08c20..cf896c0561 100644 --- a/packages/stack/tests/whole-stack/scenarios.ts +++ b/packages/stack/tests/whole-stack/scenarios.ts @@ -636,7 +636,14 @@ export const writeConfinement = (runtime: Runtime) => ] : []; // Allowed roots the named entries below cover; add one line here to extend them. - const allowedRoots = [fixture.root, fixture.locations.cacheRoot, bunTranspilerCache]; + // `podman info` asks the host's rpm which package provides each engine binary, and rpm + // keeps its database under HOME; like Podman's XDG state, that is the engine's own. + const allowedRoots = [ + fixture.root, + fixture.locations.cacheRoot, + bunTranspilerCache, + ...(runtime === "podman" ? [path.join(home, ".rpmdb")] : []), + ]; // While the stack is still running, after it has been exercised: a before/after-only // check misses anything a service writes and removes again before shutdown. const duringHome = yield* snapshotTree(home); From a0c1e2742346abbccd572c99e19e6a9de2e902c4 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 13:51:40 +0200 Subject: [PATCH 096/142] test(stack): hold the gated engine stop only once the stack is running --- .../src/StackHost.container-shutdown.integration.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index 2730dc21b5..5a041aa0f3 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -431,7 +431,9 @@ it.live.skipIf(process.platform === "win32")( /** * Installs an engine CLI shim the owner finds first on its PATH: the first `stop` announces * itself by creating `waiting` in `gateDir` and blocks until the test writes the `release` FIFO, - * then every command runs the real engine. Returns the idempotent, scope-owned `release`. + * then every command runs the real engine. The gate only holds once the test has created `armed`, + * so a one-shot container stopped during startup never takes it. Returns the idempotent, + * scope-owned `release`. */ const holdDockerStops = (gateDir: string) => Effect.gen(function* () { @@ -447,7 +449,7 @@ const holdDockerStops = (gateDir: string) => [ "#!/bin/sh", `gate='${gateDir}'`, - 'if [ ! -e "$gate/passed" ]; then', + 'if [ -e "$gate/armed" ] && [ ! -e "$gate/passed" ]; then', ' for arg in "$@"; do', ' if [ "$arg" = stop ]; then', ' : > "$gate/waiting"', @@ -551,6 +553,7 @@ const abandonsWhileStopRuns = (trigger: StopTrigger) => // Subscribes to the held engine stop's own marker before triggering the stop, so the // registration deletion below never races the gate itself. const waiting = yield* watchEntry(gateDir, "waiting", true); + yield* fs.writeFileString(`${gateDir}/armed`, ""); yield* triggerStop(trigger, access); yield* waiting.pipe(Effect.timeout("30 seconds")); From 28206e1000a5ccb8826d558c3fd88ce8572a154f Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 14:06:39 +0200 Subject: [PATCH 097/142] docs(stack): describe Podman as a container runtime of the stack --- ...017-simplified-managed-stack-architecture.md | 17 +++++++++-------- ...025-ephemeral-postgres-for-schema-tooling.md | 14 ++++++++------ packages/stack/ARCHITECTURE.md | 6 ++++-- packages/stack/README.md | 12 ++++++------ 4 files changed, 27 insertions(+), 22 deletions(-) diff --git a/docs/adr/0017-simplified-managed-stack-architecture.md b/docs/adr/0017-simplified-managed-stack-architecture.md index b02dac32e0..a4cb733990 100644 --- a/docs/adr/0017-simplified-managed-stack-architecture.md +++ b/docs/adr/0017-simplified-managed-stack-architecture.md @@ -132,14 +132,15 @@ Linux, and Windows in CI. Every managed document records one concrete runtime selection. Native and container runtimes never mix. For a new stack, an omitted runtime checks that -the Docker client is installed and then probes the daemon with a short timeout; -it selects Docker when the daemon is reachable and native otherwise. An -installed client with an unreachable daemon selects native, and the created -Effect handle carries a notice that the selection is persisted for that stack, -so a later switch to Docker requires destroying the stack or choosing a new -stack name; callers such as `stack start` decide whether to print it. Native as +each container engine in turn, Docker then Podman, resolves the same engine +target the stack owner pins and probes it with a 10 s timeout; it selects the +first reachable engine and native otherwise, where native artifacts are +supported. When Docker is skipped, the created Effect handle carries a notice +that the selected runtime (Podman or native) is persisted for that stack, so a +later switch to Docker requires destroying the stack or choosing a new stack +name; callers such as `stack start` decide whether to print it. Native as root runs only PostgreSQL as an unprivileged user. Existing state is reused -without probing. Callers may explicitly select native or Docker without +without probing. Callers may explicitly select native, Docker, or Podman without fallback. See [ADR-0025](0025-ephemeral-postgres-for-schema-tooling.md) for the daemon probe and its interaction with ephemeral Postgres. @@ -299,7 +300,7 @@ Tests follow consumed boundaries: ownership, stale-owner recovery, and interrupted cleanup; - supervisor integration covers detached ownership, RPC, stop, destroy, retirement, and wake-up; and -- one shared stack-package E2E journey runs in native and Docker modes, starts +- one shared stack-package E2E journey runs in native, Docker, and Podman modes, starts with PostgreSQL alone, activates every other service through realistic traffic, verifies cross-service behavior, then exercises stop/start and retained offline observability. diff --git a/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md b/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md index b65f561104..a3415da6ee 100644 --- a/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md +++ b/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md @@ -124,10 +124,12 @@ whether **this start created the stack identity** (first create / `unconfigured` ### Default runtime and native-as-root -Auto-selecting a **new** identity probes the Docker daemon (not only `docker --version`). A live -daemon persists Docker. A present client with a dead daemon persists **native** and prints a -notice that destroy-and-recreate (or a new `--stack` name) is required to get Docker later. -Persisted runtime never flips. Explicit `--runtime docker` still requires a live daemon. +Auto-selecting a **new** identity probes the Docker engine (not only `docker --version`), then +Podman. A reachable engine persists its runtime. When neither is reachable, **native** is persisted +where native artifacts exist. Skipping Docker prints a notice that the chosen runtime is saved with +the stack and that destroy-and-recreate (or a new `--stack` name) is required to get Docker later. +Persisted runtime never flips. Explicit `--runtime docker` or `--runtime podman` still requires a +reachable engine. Native Postgres refuses to run as uid 0 (`initdb` refuses root), so when the stack runs as root only the PostgreSQL process steps down to an unprivileged user: one named by @@ -151,7 +153,7 @@ leaving schema policy in the CLI. ### Positive -- Native and Docker shadows share one API and the same slim baseline as `stack start`. +- Native and container shadows share one API and the same slim baseline as `stack start`. - Schema commands can target a running project stack through `credentials()` when the flag is on. `credentials().database` is available whenever the database listener is assigned, including when Auth is disabled. `credentials().api` is absent when Auth is off. Overlay and `--local` keep @@ -169,7 +171,7 @@ leaving schema policy in the CLI. Compose. A failed cold launch that never reached running retries first-create. - Windows native dump/test/squash need a working Docker client even though Postgres itself is native. - Native stacks as uid 0 need `SUPABASE_NATIVE_POSTGRES_USER` or a detected agent sandbox; otherwise Docker (or a non-root user) is required. -- Auto-selected native after a dead Docker daemon is sticky until destroy or a new stack name. +- Auto-selected native or Podman after an unreachable Docker engine is sticky until destroy or a new stack name. ## Alternatives Considered diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 43bce6e1ee..ad2122c159 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -505,7 +505,9 @@ registration again, and it does not release its port reservations: a stale reser reclaimed lazily by whichever stack next needs the port. Deletion, a move and an unmount are treated identically, because a stack whose root is unreachable cannot operate regardless of which of the three caused it. -**Unreachable engine.** An owner of a container stack fails startup with a `runtime-unavailable` reason when its first container sweep finds the engine CLI missing or its daemon not listening; permission, TLS, authentication and timeout failures are ordinary startup errors. `destroy` then proceeds without an owner: it only confirms that no owner holds the stack's lease, leaves the registration, port reservations and host data untouched, and returns `{ runtimeCleanup: "skipped", engine }`; run it again once the engine is reachable. `stop` without a live owner takes the same lease and reclaims the registered stack's leftovers (the orphan-sweep path: containers, plus destruction of a session stack) before succeeding; it fails if another process holds the lease or the cleanup fails, and so needs the engine for a container stack. +**Engine targets.** A container stack pins the engine target it first resolves, so every command it runs reaches the same engine. Docker pins `DOCKER_HOST` as `--host`, else the current context. Podman pins `CONTAINER_HOST` as `--url`, else `CONTAINER_CONNECTION` as `--connection`; with neither set, a bare `podman info` decides whether the engine is local or remote. A remote engine (macOS `podman machine`) pins the default system connection by name, and a local one adds no flags. On rootless Podman, containers that run as the caller's uid with writable borrowed bind mounts get `--userns=keep-id` so files stay owned by the caller. Containers reach host listeners through the engine's host alias, `host.docker.internal` or `host.containers.internal`. + +**Unreachable engine.** An owner of a container stack fails startup with a `runtime-unavailable` reason when its first container sweep finds the engine CLI missing or its daemon or service not listening; permission, TLS, authentication and timeout failures are ordinary startup errors. `destroy` then proceeds without an owner: it only confirms that no owner holds the stack's lease, leaves the registration, port reservations and host data untouched, and returns `{ runtimeCleanup: "skipped", engine }`; run it again once the engine is reachable. `stop` without a live owner takes the same lease and reclaims the registered stack's leftovers (the orphan-sweep path: containers, plus destruction of a session stack) before succeeding; it fails if another process holds the lease or the cleanup fails, and so needs the engine for a container stack. During Serving, keep the owner alive independently of callers. Sleeping instances still need its public listeners. This is process lifetime management, not automatic service restart or continuous reconciliation. @@ -724,7 +726,7 @@ Keep the contract narrow: - Save requires confirmed stopped, initialized data. The caller explicitly stops first; snapshotting does not secretly stop dependents or restart services. A save publishes only a complete entry and replaces the previous entry for the same key. - Restore requires a confirmed stopped instance with empty data. Validate format, artifact/runtime compatibility and initialization profile before installing restored data. A missing key returns `false`; a compatible published entry returns `true`; reject a nonempty target rather than overwriting it. - Both operations hold the instance's storage reservation in the lifecycle reducer and leave it stopped. While the reservation is held, a start keeps the instance stopped until it is released and another storage operation is rejected at once. A destroy waits for the storage operation to settle, then takes the reservation; once a destroy is pending, new storage operations are refused, so it cannot be starved. No new lifecycle states are necessary; the observable current operation identifies snapshot work. An armed wake route is not a substitute for explicit stop. -- Native snapshots copy or clone the host data; container snapshots copy database data through a managed volume and helper. Docker data normally lives in a managed volume, while existing host data can be retained through the host-backed fallback. The host storage marker detects a missing or mismatched Docker volume; deleting that volume loses its database data. +- Native snapshots copy or clone the host data; container snapshots copy database data through a managed volume and helper. Docker data normally lives in a managed volume, while existing host data can be retained through the host-backed fallback; Podman always uses host-backed data. The host storage marker detects a missing or mismatched Docker volume; deleting that volume loses its database data. - Restore transfers compatible database contents, not the source instance's identity, public port claims or composition membership. The target retains its own data location and configuration, with database-specific credentials reconciled before readiness. Cache snapshots survive destruction of the source instance because their managed storage is separate; instance snapshots restore only into their own instance. These are physical database snapshots for the cache use case. A `pg_dump` invocation remains an ordinary client command for logical exports. CLI code owns cache keys, migrations and the decision to fall back to rebuilding a baseline; managed storage owns publication and retention. The snapshot API does not acquire CLI cache policy. diff --git a/packages/stack/README.md b/packages/stack/README.md index a1ffd65e39..07aef09aca 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -14,7 +14,7 @@ import { postgres } from "@supabase/stack/commands"; const stack = await create({ projectRoot: process.cwd(), - runtime: "native", // or "docker" + runtime: "native", // or "docker" or "podman" stateRoot: "/tmp/example-stack/state", cacheRoot: "/tmp/example-stack/cache", }); @@ -59,7 +59,7 @@ await stack.close(); // disconnects this client Creating a service records its definition. Configured public ports are bound during startup and retained across normal stop/start and owner reopening. An occupied saved port reports a conflict instead of moving. Automatic ports avoid every port reserved in the per-user port registry (see Public ports); a fixed port another stack holds is a conflict that names that stack. Omitted public endpoints are not exposed. -Native public listeners bind to loopback. Docker public proxies bind all interfaces so services inside the container network can reach them; those listeners are reachable from the LAN according to the host firewall. +Native public listeners bind to loopback. Container-runtime (Docker and Podman) public proxies bind all interfaces so services inside the container network can reach them; those listeners are reachable from the LAN according to the host firewall. Functions use a package-provided, self-contained Edge Runtime main service unless the configuration supplies `bootstrap` source; only an explicit `bootstrap` is saved with the service definition. Database versions belong in `config.version`; other recipes accept an optional top-level artifact `version`. @@ -77,9 +77,9 @@ Pass `startOwner: true` to `open` when live status and other owner-backed operat `create({ ..., startOwner: true })` starts a detached stack's owner immediately and lets it register the stack under its lease, as a session stack always does. If the owner fails to start or the launch is interrupted before it reports ready, the owner removes that registration and `create` fails with the launch error, so a failed launch leaves no stack behind. -`destroy` normally returns `{ runtimeCleanup: "complete" }`. When no owner is running and a new owner cannot start because the stack's container engine reports that its daemon cannot be reached, `destroy` confirms that no owner holds the stack's lease and leaves the registration, port reservations and host data untouched, returning `{ runtimeCleanup: "skipped", engine }`. When an owner does run but a labelled container cannot be removed (for example one on a different daemon) or an instance's data cannot be removed, `destroy` likewise keeps the stack registered and fails, listing what remains. Either way, retry `destroy` once the engine is reachable; there is no `cleanupCommands` field. +`destroy` normally returns `{ runtimeCleanup: "complete" }`. When no owner is running and a new owner cannot start because the stack's container engine (Docker or Podman) cannot be reached, `destroy` confirms that no owner holds the stack's lease and leaves the registration, port reservations and host data untouched, returning `{ runtimeCleanup: "skipped", engine }`. When an owner does run but a labelled container cannot be removed (for example one on a different daemon) or an instance's data cannot be removed, `destroy` likewise keeps the stack registered and fails, listing what remains. Either way, retry `destroy` once the engine is reachable; there is no `cleanupCommands` field. -The stack owns database, Functions bootstrap, and command-job directories below its data directory. Storage uploads remain at the caller-supplied Storage `filePath` and are preserved when the stack is destroyed; the caller owns that directory. Host metadata remains under `stateRoot`; native database data uses host files. Docker database data normally uses a managed volume, while existing host data is retained through the host-backed fallback. A host marker records the selected Docker storage and detects a missing or mismatched volume; deleting that volume loses the associated database data. Native snapshot entries live below `cacheRoot`. Docker snapshots share the managed data volume in a separate namespace derived from `cacheRoot`, so they survive source destruction and can use filesystem cloning. A Docker cache hit requires the same daemon, `stateRoot`, and `cacheRoot`. There is no portable tar snapshot API. +The stack owns database, Functions bootstrap, and command-job directories below its data directory. Storage uploads remain at the caller-supplied Storage `filePath` and are preserved when the stack is destroyed; the caller owns that directory. Host metadata remains under `stateRoot`; native database data uses host files. Docker database data normally uses a managed volume, while existing host data is retained through the host-backed fallback; Podman always keeps database data in a host-backed directory. A host marker records the selected Docker storage and detects a missing or mismatched volume; deleting that volume loses the associated database data. Native snapshot entries live below `cacheRoot`. Docker snapshots share the managed data volume in a separate namespace derived from `cacheRoot`, so they survive source destruction and can use filesystem cloning. A Docker cache hit requires the same daemon, `stateRoot`, and `cacheRoot`. There is no portable tar snapshot API. Omitted database `jwtSecret` and `rootKey` inputs use the shared local-development values exported as `DEFAULT_LOCAL_JWT_SECRET` and `DEFAULT_POSTGRES_ROOT_KEY`. Explicit values override these defaults. @@ -150,7 +150,7 @@ Inputs that the composition binds or the owner fills from the stack credentials, To recompose a stopped stack, pass the IDs to keep in `reuseIds`. `composition.plan(services)` compares requested creations with every saved instance of the same kinds without contacting the owner or changing state. Each entry reports its instance `id`, `service`, whether it is a composition `member`, and a `change`: `unchanged`, `changed` with the differing config `paths`, or `incompatible` with the `paths` of an endpoint, artifact version, or PostgreSQL major-version change that the saved instance cannot adopt. Inputs that the composition or the stack credentials supply are ignored, an automatic API port is compared as the shared fixed port the composition would assign, and a PostgreSQL major alias matches its pinned version. Recomposing with `reuseIds` replaces a `changed` instance's configuration while keeping its identity, data, and ports. -The whole-stack E2E suite covers the default lazy lifecycle and reopen, all-eager startup, idle and wake, and parallel stack isolation for native and Docker runtimes, split into a `lifecycle` and an `idle-parallel` file per runtime. Run one runtime with a filter such as `pnpm --filter @supabase/stack test:e2e:run src/whole-stack.native` (matches both native files) or target one file, for example `src/whole-stack.docker.idle-parallel.e2e.test.ts`. +The whole-stack E2E suite covers the default lazy lifecycle and reopen, all-eager startup, idle and wake, and parallel stack isolation for native, Docker, and Podman runtimes, split into a `lifecycle` and an `idle-parallel` file per runtime. The Podman files need `SUPABASE_STACK_TEST_ENGINE=podman`; CI runs them, and the container integration suite, on rootless Podman 4.9 on Linux. macOS `podman machine` is not covered by CI. Run one runtime with a filter such as `pnpm --filter @supabase/stack test:e2e:run src/whole-stack.native` (matches both native files) or target one file, for example `src/whole-stack.docker.idle-parallel.e2e.test.ts`. For multiple instances or custom dependencies, configure members and bindings explicitly instead: @@ -206,7 +206,7 @@ await test.reset("seeded"); // discard writes made since the checkpoint The stack is a session stack composed with `composition.supabase` and `{ eager: true }`; each selected service is configured for local development with every endpoint on an automatic port, and `test.services` holds a typed handle per selected kind. Select a kind by name, or pass `{ service, config, endpoints }` to override part of its configuration. Disposal destroys the stack, then closes the client and removes the temporary project root. A session stack is also destroyed when the test process exits. -Every test stack for the current OS user shares one state root and the package's artifact cache root under the OS temp directory, so their ports are coordinated and Docker uses one data volume; each gets a unique temporary project root and name. Pass `stateRoot`, `cacheRoot`, or `projectRoot` to override them. The runtime comes from `runtime`, then `SUPABASE_STACK_TEST_RUNTIME`, then the platform default: native where the catalog publishes native artifacts and Docker elsewhere. A startup failure names the state of each registered service and the owner log. +Every test stack for the current OS user shares one state root and the package's artifact cache root under the OS temp directory, so their ports are coordinated and Docker uses one data volume (Podman uses host directories); each gets a unique temporary project root and name. Pass `stateRoot`, `cacheRoot`, or `projectRoot` to override them. The runtime comes from `runtime`, then `SUPABASE_STACK_TEST_RUNTIME`, then the platform default: native where the catalog publishes native artifacts and Docker elsewhere. A startup failure names the state of each registered service and the owner log. `checkpoint(name)` stops the composition, saves the database data as an instance-scoped snapshot, and starts the composition again. `reset(name)` stops it, clears the database data, restores that snapshot, and waits until every service is ready. Checkpoints are never evicted by other stacks' snapshots and are removed when the test stack is destroyed. From db671fd8fc2ad6bab444a71035f4861d28e4d472 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 16:07:25 +0200 Subject: [PATCH 098/142] test(stack): list Podman workloads through its JSON output, stable across Podman versions --- packages/stack/tests/whole-stack/workloads.ts | 57 ++++++++++++------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/packages/stack/tests/whole-stack/workloads.ts b/packages/stack/tests/whole-stack/workloads.ts index abf144b567..ae29dcf0d4 100644 --- a/packages/stack/tests/whole-stack/workloads.ts +++ b/packages/stack/tests/whole-stack/workloads.ts @@ -10,6 +10,15 @@ export const WorkloadSnapshot = Schema.Struct({ }); export type WorkloadSnapshot = Schema.Schema.Type; +const PodmanContainers = Schema.fromJsonString( + Schema.Array( + Schema.Struct({ + Id: Schema.String, + Labels: Schema.Record(Schema.String, Schema.String), + }), + ), +); + export const commandOutput = Effect.fn("WholeStack.commandOutput")( (command: string, args: ReadonlyArray) => Effect.gen(function* () { @@ -37,25 +46,35 @@ const snapshotOutput = (runtime: Runtime, fixture: WholeStack, includeStopped: b } satisfies WorkloadSnapshot; } const marker = `com.supabase.stack=${fixture.stack.id}`; - // Docker's `ps` template reads a label through `.Label`; Podman's has no such method. - const label = (key: string) => - runtime === "docker" ? `{{.Label "${key}"}}` : `{{index .Labels "${key}"}}`; - const output = yield* commandOutput(runtime, [ - "ps", - ...(includeStopped ? ["--all"] : []), - "--filter", - `label=${marker}`, - "--format", - `{{.ID}}\t${label("com.supabase.instance")}\t${label("com.supabase.stack-managed")}`, - ]); - const records = output - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => { - const [id, instance, managed] = line.split("\t"); - return { identity: `${id ?? ""} ${instance ?? ""}`.trim(), instance, managed }; - }) + const listArgs = ["ps", ...(includeStopped ? ["--all"] : []), "--filter", `label=${marker}`]; + // Podman's `ps` templates read labels differently across versions; its JSON is stable. + const listed = + runtime === "docker" + ? (yield* commandOutput(runtime, [ + ...listArgs, + "--format", + '{{.ID}}\t{{.Label "com.supabase.instance"}}\t{{.Label "com.supabase.stack-managed"}}', + ])) + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => { + const [id, instance, managed] = line.split("\t"); + return { id, instance, managed }; + }) + : (yield* Schema.decodeEffect(PodmanContainers)( + yield* commandOutput(runtime, [...listArgs, "--format", "json"]), + )).map(({ Id, Labels }) => ({ + id: Id, + instance: Labels["com.supabase.instance"], + managed: Labels["com.supabase.stack-managed"], + })); + const records = listed + .map(({ id, instance, managed }) => ({ + identity: `${id ?? ""} ${instance ?? ""}`.trim(), + instance, + managed, + })) // The stack label alone also tags best-effort containers that carry neither an instance // nor a `stack-managed` identity, such as the host-gateway probe (see `readProbeHosts` in // `Container.ts`); only the two documented identity labels promise a workload. From 3391de273c9e3604007035d078e0baf15ee1fd5b Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 16:32:08 +0200 Subject: [PATCH 099/142] test(stack): treat the Podman client's SSH and runtime state as its own in the confinement check --- packages/stack/tests/whole-stack/scenarios.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/stack/tests/whole-stack/scenarios.ts b/packages/stack/tests/whole-stack/scenarios.ts index cf896c0561..63acd98d57 100644 --- a/packages/stack/tests/whole-stack/scenarios.ts +++ b/packages/stack/tests/whole-stack/scenarios.ts @@ -636,13 +636,20 @@ export const writeConfinement = (runtime: Runtime) => ] : []; // Allowed roots the named entries below cover; add one line here to extend them. - // `podman info` asks the host's rpm which package provides each engine binary, and rpm - // keeps its database under HOME; like Podman's XDG state, that is the engine's own. + // Podman's own client state: `podman info` asks the host's rpm which package provides each + // engine binary (rpm's database), a remote machine connection is SSH, and without + // XDG_RUNTIME_DIR (macOS) the client keeps its runtime directory under TMPDIR. const allowedRoots = [ fixture.root, fixture.locations.cacheRoot, bunTranspilerCache, - ...(runtime === "podman" ? [path.join(home, ".rpmdb")] : []), + ...(runtime === "podman" + ? [ + path.join(home, ".rpmdb"), + path.join(home, ".ssh"), + path.join(tmp, `storage-run-${process.getuid?.()}`), + ] + : []), ]; // While the stack is still running, after it has been exercised: a before/after-only // check misses anything a service writes and removes again before shutdown. From 11e56f518649aecf98ca07a44ab0cd3657d4d548 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 16:32:09 +0200 Subject: [PATCH 100/142] docs(stack): note the podman machine memory a full stack needs on macOS --- packages/stack/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/stack/README.md b/packages/stack/README.md index 07aef09aca..47593c3add 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -61,6 +61,8 @@ Creating a service records its definition. Configured public ports are bound dur Native public listeners bind to loopback. Container-runtime (Docker and Podman) public proxies bind all interfaces so services inside the container network can reach them; those listeners are reachable from the LAN according to the host firewall. +On macOS, Podman runs containers in a `podman machine` VM. A full stack needs about 8 GiB of VM memory (`podman machine set --memory 8192`); the 2 GiB default runs out of memory and the kernel kills services. + Functions use a package-provided, self-contained Edge Runtime main service unless the configuration supplies `bootstrap` source; only an explicit `bootstrap` is saved with the service definition. Database versions belong in `config.version`; other recipes accept an optional top-level artifact `version`. On Linux, native Functions project files must be outside `/tmp`: Edge Runtime uses a private filesystem at that path. Docker mounts project files at a separate runtime path. From 7a51475ff226c8d4dc2b409ad7ae1756230373e6 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 18:42:23 +0200 Subject: [PATCH 101/142] ci(stack): share the rootless Podman setup, stress Podman, and run the port reservation e2e on Linux --- .github/actions/setup-podman/action.yml | 23 ++++++++++++++ .github/workflows/stack-stress.yml | 8 ++++- .github/workflows/test.yml | 42 ++++++------------------- 3 files changed, 39 insertions(+), 34 deletions(-) create mode 100644 .github/actions/setup-podman/action.yml diff --git a/.github/actions/setup-podman/action.yml b/.github/actions/setup-podman/action.yml new file mode 100644 index 0000000000..ee30ca3603 --- /dev/null +++ b/.github/actions/setup-podman/action.yml @@ -0,0 +1,23 @@ +name: Setup Podman +description: Install rootless Podman and prepare its per-user runtime directory + +runs: + using: composite + steps: + # Rootless Podman needs subordinate id ranges and a per-user runtime directory; the user + # service is started explicitly so /run/user/ exists before the first podman call. + - name: Prepare rootless Podman + shell: bash + run: | + if ! command -v podman >/dev/null; then + sudo apt-get update + sudo apt-get install -y podman + fi + user="$(id -un)" + grep -q "^${user}:" /etc/subuid || sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 "${user}" + sudo loginctl enable-linger "${user}" + sudo systemctl start "user@$(id -u).service" + echo "XDG_RUNTIME_DIR=/run/user/$(id -u)" >> "$GITHUB_ENV" + XDG_RUNTIME_DIR="/run/user/$(id -u)" podman system migrate + podman version + podman info diff --git a/.github/workflows/stack-stress.yml b/.github/workflows/stack-stress.yml index 3f7cdcb3de..2d10715a3f 100644 --- a/.github/workflows/stack-stress.yml +++ b/.github/workflows/stack-stress.yml @@ -23,10 +23,12 @@ jobs: name: Run stack stress scenarios (${{ matrix.runtime }}) runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 90 + env: + SUPABASE_STACK_TEST_ENGINE: ${{ matrix.runtime == 'podman' && 'podman' || 'docker' }} strategy: fail-fast: false matrix: - runtime: [native, docker] + runtime: [native, docker, podman] steps: - name: Checkout uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1.0.0-beta @@ -37,6 +39,10 @@ jobs: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} go-cache: "false" + - name: Setup Podman + if: matrix.runtime == 'podman' + uses: ./.github/actions/setup-podman + # vitest's --repeats counts the runs after the first. - name: Run idle and parallel-stack scenarios repeatedly working-directory: packages/stack diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 403627f486..f8958f7d39 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -127,7 +127,11 @@ jobs: # registry cross-process and publication-crash scenarios run on Linux too. - name: Run stack namespace e2e tests working-directory: packages/stack - run: pnpm test:e2e:run src/namespace/Namespace.e2e.test.ts --passWithNoTests=false + run: >- + pnpm test:e2e:run + src/namespace/Namespace.e2e.test.ts + src/port-reservation.e2e.test.ts + --passWithNoTests=false test-stack-ports: if: | @@ -340,23 +344,9 @@ jobs: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} go-cache: "false" - # Rootless Podman needs subordinate id ranges and a per-user runtime directory; the user - # service is started explicitly so /run/user/ exists before the first podman call. - - name: Prepare rootless Podman + - name: Setup Podman if: matrix.runtime == 'podman' - run: | - if ! command -v podman >/dev/null; then - sudo apt-get update - sudo apt-get install -y podman - fi - user="$(id -un)" - grep -q "^${user}:" /etc/subuid || sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 "${user}" - sudo loginctl enable-linger "${user}" - sudo systemctl start "user@$(id -u).service" - echo "XDG_RUNTIME_DIR=/run/user/$(id -u)" >> "$GITHUB_ENV" - XDG_RUNTIME_DIR="/run/user/$(id -u)" podman system migrate - podman version - podman info + uses: ./.github/actions/setup-podman - name: Run stack end-to-end tests run: >- @@ -391,22 +381,8 @@ jobs: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} go-cache: "false" - # Rootless Podman needs subordinate id ranges and a per-user runtime directory; the user - # service is started explicitly so /run/user/ exists before the first podman call. - - name: Prepare rootless Podman - run: | - if ! command -v podman >/dev/null; then - sudo apt-get update - sudo apt-get install -y podman - fi - user="$(id -un)" - grep -q "^${user}:" /etc/subuid || sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 "${user}" - sudo loginctl enable-linger "${user}" - sudo systemctl start "user@$(id -u).service" - echo "XDG_RUNTIME_DIR=/run/user/$(id -u)" >> "$GITHUB_ENV" - XDG_RUNTIME_DIR="/run/user/$(id -u)" podman system migrate - podman version - podman info + - name: Setup Podman + uses: ./.github/actions/setup-podman # The suites that drive a real container engine; the rest of the integration project is # engine-independent and runs in test-integration. From 614bdb2bc0e1ec723f6cbdd057a197a16721dd57 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 18:42:25 +0200 Subject: [PATCH 102/142] refactor(stack): check only the native runtime base directory's ownership --- .../runtime/postgres-user.integration.test.ts | 42 +++++++++---------- packages/stack/src/runtime/postgres-user.ts | 10 ++--- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/packages/stack/src/runtime/postgres-user.integration.test.ts b/packages/stack/src/runtime/postgres-user.integration.test.ts index a6f867c3d2..bc6d75689c 100644 --- a/packages/stack/src/runtime/postgres-user.integration.test.ts +++ b/packages/stack/src/runtime/postgres-user.integration.test.ts @@ -119,29 +119,25 @@ describe("acquireNativeRuntimeRoot", () => { ).pipe(Effect.provide(NodeServices.layer)), ); - it.live( - "refuses a base beneath a group- or world-writable directory without the sticky bit", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const shared = yield* fs.makeTempDirectoryScoped({ - prefix: "native-runtime-root-shared-", - }); - yield* fs.chmod(shared, 0o777); - const base = path.join(shared, "runtime"); - yield* fs.makeDirectory(base, { mode: 0o700 }); + it.live("refuses a base that is group- or world-writable without the sticky bit", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const base = yield* fs.makeTempDirectoryScoped({ + prefix: "native-runtime-root-shared-", + }); + yield* fs.chmod(base, 0o777); - const failure = yield* acquireNativeRuntimeRoot().pipe( - Effect.flip, - Effect.provide(Layer.succeed(NativeRuntimeRootBase, base)), - ); - expect(failure.message).toContain("writable by group or others without the sticky bit"); - expect(yield* fs.exists(nativeRuntimeRootPath(path, base, process.getuid?.() ?? 0))).toBe( - false, - ); - }), - ).pipe(Effect.provide(NodeServices.layer)), + const failure = yield* acquireNativeRuntimeRoot().pipe( + Effect.flip, + Effect.provide(Layer.succeed(NativeRuntimeRootBase, base)), + ); + expect(failure.message).toContain("writable by group or others without the sticky bit"); + expect(yield* fs.exists(nativeRuntimeRootPath(path, base, process.getuid?.() ?? 0))).toBe( + false, + ); + }), + ).pipe(Effect.provide(NodeServices.layer)), ); }); diff --git a/packages/stack/src/runtime/postgres-user.ts b/packages/stack/src/runtime/postgres-user.ts index bf86c88f52..1c82465fe1 100644 --- a/packages/stack/src/runtime/postgres-user.ts +++ b/packages/stack/src/runtime/postgres-user.ts @@ -202,12 +202,10 @@ const resolveNativeRuntimeRoot = (create: boolean) => new ServiceError({ operation: "launch", message: `Unable to resolve ${base}`, cause }), ), ); - // Every ancestor is checked too: a writable non-sticky one would let another uid rename the tree. - for (const directory of ancestorsOf(path, canonicalBase)) - yield* confirmPrivateDirectory(directory, uid, { - allowRootOwner: true, - allowStickyWritable: true, - }); + yield* confirmPrivateDirectory(canonicalBase, uid, { + allowRootOwner: true, + allowStickyWritable: true, + }); const root = nativeRuntimeRootPath(path, canonicalBase, uid); if (create) yield* fs.makeDirectory(root, { mode: 0o700 }).pipe( From 93c734f192092676bd37bf70b53547c1ead05a65 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 18:42:26 +0200 Subject: [PATCH 103/142] docs(stack): describe reservations left by a failed individual-service destroy --- packages/stack/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/stack/README.md b/packages/stack/README.md index 47593c3add..898e06a740 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -106,8 +106,10 @@ field names the port, the endpoint, and either the live holder (`{ stackId, stat `"foreign"` for a process outside the registry. A fixed or previously saved port is never silently reassigned; only automatic allocation tries another candidate. A reservation is released when deleting an individual service releases the endpoints only it owned, or when a stack-wide `destroy` -has removed the registration; a failed `destroy` keeps every reservation, and stopping a stack or killing its -owner leaves them in place. A reservation whose owning stack's `state.json` is confirmed gone (`ENOENT`, for example +has removed the registration. A failed stack-wide `destroy` keeps every reservation; a failed +individual-service destroy may already have released some of that service's endpoints, and the +service can stay unavailable until the destroy is retried. Stopping a stack or killing its +owner leaves reservations in place. A reservation whose owning stack's `state.json` is confirmed gone (`ENOENT`, for example after deleting its state root) is reclaimed lazily by the next stack that needs its port. Known limitations: on macOS, BSD, and Windows, where the kernel allows overlapping binds, a loopback From a44dcde0d701b95e893f1b6e8cb6e75a3fdac2a6 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 18:45:24 +0200 Subject: [PATCH 104/142] refactor(stack): run the short-poll owner fixture through runHostProcess --- packages/stack/src/internal/host-process.ts | 26 ++++- .../stack/src/runtime/Session.unit.test.ts | 25 ++++- packages/stack/src/services/Database.ts | 19 +--- .../stack/src/services/Database.unit.test.ts | 22 ---- .../tests/short-registration-poll-fixture.ts | 103 +----------------- 5 files changed, 53 insertions(+), 142 deletions(-) delete mode 100644 packages/stack/src/services/Database.unit.test.ts diff --git a/packages/stack/src/internal/host-process.ts b/packages/stack/src/internal/host-process.ts index ddab65508d..395218ffea 100644 --- a/packages/stack/src/internal/host-process.ts +++ b/packages/stack/src/internal/host-process.ts @@ -1,8 +1,17 @@ -import { Cause, Effect, Exit, Option, Schema } from "effect"; +import { Cause, Duration, Effect, Exit, identity, Option, Schema } from "effect"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- readiness is an inherited launcher descriptor. import { closeSync, writeSync } from "node:fs"; import { SavedStack } from "../StackNamespace.ts"; -import { runStackHost, StackHostError, type StackHostOptions } from "../StackHost.ts"; +import { + RegistrationCheckInterval, + runStackHost, + StackHostError, + type StackHostOptions, +} from "../StackHost.ts"; + +interface HostProcessOverrides extends Pick { + readonly registrationCheckInterval?: Duration.Input; +} const writeLine = (value: unknown) => Effect.gen(function* () { @@ -62,7 +71,10 @@ const options = ( }; }); -const program = (args: ReadonlyArray, overrides: Pick) => +const program = ( + args: ReadonlyArray, + { registrationCheckInterval, ...overrides }: HostProcessOverrides, +) => Effect.gen(function* () { let reported = false; const report = (value: unknown) => @@ -82,7 +94,11 @@ const program = (args: ReadonlyArray, overrides: Pick Effect.callback((resume) => { @@ -96,7 +112,7 @@ const flushed = (stream: NodeJS.WriteStream) => */ export const runHostProcess = ( args: ReadonlyArray, - overrides: Pick = {}, + overrides: HostProcessOverrides = {}, ): Promise => Effect.runPromise( program(args, overrides).pipe( diff --git a/packages/stack/src/runtime/Session.unit.test.ts b/packages/stack/src/runtime/Session.unit.test.ts index fcf290afef..c95d3d6249 100644 --- a/packages/stack/src/runtime/Session.unit.test.ts +++ b/packages/stack/src/runtime/Session.unit.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from "@effect/vitest"; -import { Cause } from "effect"; -import { mapToServiceError } from "./Session.ts"; +import { Cause, Effect, Exit, Fiber, Option, Ref } from "effect"; +import * as TestClock from "effect/testing/TestClock"; +import { mapToServiceError, processExit } from "./Session.ts"; + +const describeExit = (code: number) => `runtime exited with code ${code}`; describe("mapToServiceError", () => { it("names an error whose cause carries no message", () => { @@ -16,3 +19,21 @@ describe("mapToServiceError", () => { expect(failure.message).toBe("container is gone"); }); }); + +it.effect("waits for the stderr tail after a long-running runtime exits", () => + Effect.gen(function* () { + const tail = yield* Ref.make(""); + const drained = yield* Effect.forkChild( + Effect.sleep("2500 millis").pipe(Effect.andThen(Ref.set(tail, "FATAL: data directory"))), + ); + const settled = yield* Effect.forkChild( + processExit(Effect.sleep("2 seconds").pipe(Effect.as(1)), describeExit, { tail, drained }), + ); + yield* TestClock.adjust("3 seconds"); + const exit = yield* Fiber.join(settled); + const error = Exit.isFailure(exit) + ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) + : undefined; + expect(error?.message).toBe("runtime exited with code 1: FATAL: data directory"); + }), +); diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index b815b2cc3b..2d541168b1 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -5,8 +5,6 @@ import { Crypto, Data, Effect, - Exit, - Fiber, FileSystem, Layer, Path, @@ -43,7 +41,7 @@ import { } from "../runtime/PostgresDatabaseSession.ts"; import { mapToServiceError, - processExit as sharedProcessExit, + processExit, publishProcessLogs, runtimeSessionFromContainer, } from "../runtime/Session.ts"; @@ -191,16 +189,6 @@ const databaseError = (operation: string, cause: unknown): DatabaseError => const describePostgresExit = (code: number) => `PostgreSQL exited with code ${code}`; -/** Settles a native PostgreSQL exit, waiting briefly after it for the stderr tail to drain. */ -export const processExit = ( - exitCode: Effect.Effect, - stderr?: { - readonly tail: Ref.Ref; - readonly drained: Fiber.Fiber; - }, -): Effect.Effect> => - sharedProcessExit(exitCode, describePostgresExit, stderr); - /** Exported for the cross-cutting pin test in {@link "../runtime/Container.integration.test.ts"}. */ export const reconcileContainerPassword = Effect.fn("Database.reconcileContainerPassword")( function* ( @@ -890,7 +878,10 @@ export const makeDatabase = ( return { health: setup, probe: setup, - exit: processExit(process.exitCode, { tail: stderrTail, drained: stderrDrained }), + exit: processExit(process.exitCode, describePostgresExit, { + tail: stderrTail, + drained: stderrDrained, + }), stop: process.kill.pipe(Effect.mapError((cause) => errorFor("stop", cause))), remove: fs.remove(socketPath, { recursive: true, force: true }).pipe( Effect.mapError((cause) => errorFor("remove", cause)), diff --git a/packages/stack/src/services/Database.unit.test.ts b/packages/stack/src/services/Database.unit.test.ts deleted file mode 100644 index 0b0cee5ce8..0000000000 --- a/packages/stack/src/services/Database.unit.test.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Fiber, Option, Ref } from "effect"; -import * as TestClock from "effect/testing/TestClock"; -import { processExit } from "./Database.ts"; - -it.effect("waits for the stderr tail after a long-running native PostgreSQL exits", () => - Effect.gen(function* () { - const tail = yield* Ref.make(""); - const drained = yield* Effect.forkChild( - Effect.sleep("2500 millis").pipe(Effect.andThen(Ref.set(tail, "FATAL: data directory"))), - ); - const settled = yield* Effect.forkChild( - processExit(Effect.sleep("2 seconds").pipe(Effect.as(1)), { tail, drained }), - ); - yield* TestClock.adjust("3 seconds"); - const exit = yield* Fiber.join(settled); - const error = Exit.isFailure(exit) - ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) - : undefined; - expect(error?.message).toBe("PostgreSQL exited with code 1: FATAL: data directory"); - }), -); diff --git a/packages/stack/tests/short-registration-poll-fixture.ts b/packages/stack/tests/short-registration-poll-fixture.ts index c6e2cf924c..052241dd31 100644 --- a/packages/stack/tests/short-registration-poll-fixture.ts +++ b/packages/stack/tests/short-registration-poll-fixture.ts @@ -1,100 +1,5 @@ -import { Cause, Effect, Exit, Option, Schema } from "effect"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- readiness is an inherited launcher descriptor. -import { closeSync, writeSync } from "node:fs"; -import { SavedStack } from "../src/StackNamespace.ts"; -import { RegistrationCheckInterval, runStackHost, StackHostError } from "../src/StackHost.ts"; +import { runHostProcess } from "../src/internal/host-process.ts"; -/** - * A real owner entrypoint, identical to `internal/host-process.ts`, except it overrides the - * registration-loss poll interval through the sanctioned internal `Context.Reference` so an - * abandonment test does not wait a real 30 seconds. Production code never reads this from an - * environment variable or `Config`; only this dedicated test fixture supplies a different value. - */ -const writeLine = (value: unknown) => - Effect.gen(function* () { - const serialized = yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))( - value, - ).pipe( - Effect.mapError( - (cause) => new StackHostError({ operation: "ready", message: String(cause), cause }), - ), - ); - yield* Effect.try({ - try: () => writeSync(3, Buffer.from(`${serialized}\n`, "utf8")), - catch: (cause) => new StackHostError({ operation: "ready", message: String(cause), cause }), - }); - }).pipe( - Effect.ensuring( - Effect.sync(() => { - try { - closeSync(3); - } catch { - // The parent may close the inherited descriptor during cancellation. - } - }), - ), - ); - -const [stateRoot, cacheRoot, stackId, register, ...rest] = process.argv.slice(2); - -const program = Effect.gen(function* () { - if ( - stateRoot === undefined || - cacheRoot === undefined || - stackId === undefined || - rest.length > 0 - ) - return yield* new StackHostError({ - operation: "startup", - message: "Expected stateRoot, cacheRoot, stackId and an optional stack to register", - }); - const registered = - register === undefined - ? undefined - : yield* Schema.decodeEffect(Schema.fromJsonString(SavedStack))(register).pipe( - Effect.mapError( - (cause) => new StackHostError({ operation: "startup", message: cause.message }), - ), - ); - let reported = false; - const report = (value: unknown) => - Effect.suspend(() => { - if (reported) return Effect.void; - reported = true; - return writeLine(value); - }); - yield* runStackHost({ - stateRoot, - cacheRoot, - stackId, - ...(registered === undefined ? {} : { register: registered }), - onReady: ({ endpoint, secret }) => report({ type: "ready", endpoint, secret }), - }).pipe( - Effect.catchCause((cause) => { - const failure = Option.getOrUndefined(Cause.findErrorOption(cause)); - return report({ - type: "error", - message: failure?.message ?? Cause.pretty(cause), - ...(failure?.reason === undefined ? {} : { reason: failure.reason }), - }).pipe(Effect.exit, Effect.andThen(Effect.failCause(cause))); - }), - ); -}).pipe(Effect.provideService(RegistrationCheckInterval, "200 millis")); - -const flushed = (stream: NodeJS.WriteStream) => - Effect.callback((resume) => { - stream.write("", () => resume(Effect.void)); - }).pipe(Effect.timeoutOption("1 second")); - -await Effect.runPromise( - program.pipe( - Effect.exit, - Effect.tap((exit) => - Exit.isFailure(exit) - ? Effect.sync(() => process.stderr.write(`${Cause.pretty(exit.cause)}\n`)) - : Effect.void, - ), - Effect.tap(() => Effect.all([flushed(process.stdout), flushed(process.stderr)])), - Effect.flatMap((exit) => Effect.sync(() => process.exit(Exit.isSuccess(exit) ? 0 : 1))), - ), -); +/** An owner whose registration-loss poll runs every 200ms, so abandonment tests need not wait the production interval. */ +if (import.meta.main) + await runHostProcess(process.argv.slice(2), { registrationCheckInterval: "200 millis" }); From 0c7249f68c58ddb060f547d44874f64c6b385387 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 18:45:26 +0200 Subject: [PATCH 105/142] test(stack): drop tests whose contracts are covered elsewhere --- packages/stack/src/Artifacts.unit.test.ts | 11 +- packages/stack/src/Lifecycle.unit.test.ts | 125 ------------------ .../src/Orchestrator.integration.test.ts | 109 --------------- .../stack/src/Service.integration.test.ts | 27 ---- .../drivers/Sqlite.integration.test.ts | 28 ---- .../stack/src/port-reservation.e2e.test.ts | 2 +- .../stack/src/runtime/Container.unit.test.ts | 52 +------- .../src/runtime/ContainerName.unit.test.ts | 20 --- .../DockerDatabaseStorage.integration.test.ts | 84 ------------ 9 files changed, 3 insertions(+), 455 deletions(-) delete mode 100644 packages/stack/src/namespace/drivers/Sqlite.integration.test.ts diff --git a/packages/stack/src/Artifacts.unit.test.ts b/packages/stack/src/Artifacts.unit.test.ts index 50c55fd78c..0e0ecf510d 100644 --- a/packages/stack/src/Artifacts.unit.test.ts +++ b/packages/stack/src/Artifacts.unit.test.ts @@ -1,14 +1,5 @@ import { expect, it } from "@effect/vitest"; -import { catalogPins, slimImageMirrors } from "./Artifacts.ts"; - -it("carries a normalized upstreamImage for every catalog pin", () => { - for (const { pin } of catalogPins()) { - expect(pin.upstreamImage).not.toBe(""); - expect(pin.upstreamImage).not.toContain("docker.io/"); - expect(pin.upstreamImage).not.toContain("@sha256:"); - expect(pin.upstreamImage.split(":").at(-1)).not.toBe(""); - } -}); +import { slimImageMirrors } from "./Artifacts.ts"; it("rewrites a GHCR catalog image onto ECR Public and keeps the tag and digest", () => { expect( diff --git a/packages/stack/src/Lifecycle.unit.test.ts b/packages/stack/src/Lifecycle.unit.test.ts index 3b17db2069..f3f0b9339a 100644 --- a/packages/stack/src/Lifecycle.unit.test.ts +++ b/packages/stack/src/Lifecycle.unit.test.ts @@ -80,26 +80,6 @@ const commandTagged = ( }; describe("Lifecycle reducer", () => { - it("wakes a lazy service on first traffic, admits the waiter once ready, and sleeps after idle", () => { - const graph = makeGraph([lazy("api")]); - const { commandsByStep, state } = run(initialState(graph), [ - { event: open("api", 1), now: 0 }, - { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, - { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 20 }, - ]); - - expect(tagsOf(commandsByStep[0])).toEqual(["Launch"]); - expect(tagsOf(commandsByStep[1])).toEqual(["AdmitConnection"]); - expect(tagsOf(commandsByStep[2])).toEqual(["ArmIdleTimer"]); - const armed = at(at(commandsByStep, 2), 0) as Extract; - expect(armed).toMatchObject({ id: "api", generation: 1, epoch: 1, delayMillis: 1_000 }); - expect(state.services.get("api")?.phase).toEqual({ - _tag: "Running", - generation: 1, - ready: true, - }); - }); - it("ignores a launch outcome, an idle elapse and an exit tagged with a stale generation", () => { const graph = makeGraph([lazy("api")]); const { state, commandsByStep } = run(initialState(graph), [ @@ -490,47 +470,6 @@ describe("Lifecycle reducer", () => { expect(loop.services.get("flaky")?.breaker.openUntil).toBeDefined(); }); - it("reopens for the same fixed cooldown on a further failure, and an explicit start resets it", () => { - const graph = makeGraph([lazy("flaky")]); - let state = initialState(graph); - - const fail = (waiterId: number, now: number) => { - ({ state } = run(state, [{ event: open("flaky", waiterId), now }])); - const generation = startingGeneration(state, "flaky"); - ({ state } = run(state, [ - { - event: LifecycleEvent.LaunchFailed({ id: "flaky", generation, cause: "boom" }), - now: now + 1, - }, - ])); - }; - - fail(1, 0); - fail(2, 1); - fail(3, 2); - expect(state.services.get("flaky")?.breaker.openUntil).toBe(3 + 30_000); - - ({ state } = run(state, [ - { - event: LifecycleEvent.CooldownElapsed({ id: "flaky", openUntil: 3 + 30_000 }), - now: 3 + 30_000, - }, - ])); - fail(4, 3 + 30_000); - expect(state.services.get("flaky")?.breaker.consecutiveFailures).toBe(4); - expect(state.services.get("flaky")?.breaker.openUntil).toBe(3 + 30_000 + 1 + 30_000); - - ({ state } = run( - state, - [{ event: LifecycleEvent.StartRequested({ id: "flaky" }) }].map((step) => ({ - ...step, - now: 500_000, - })), - )); - expect(state.services.get("flaky")?.breaker.consecutiveFailures).toBe(0); - expect(state.services.get("flaky")?.breaker.openUntil).toBeUndefined(); - }); - it("schedules a cooldown timer when the breaker opens and only the dedicated event retries, even with demand retained the whole time", () => { const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); let state = initialState(graph); @@ -601,36 +540,6 @@ describe("Lifecycle reducer", () => { }); }); - it("caps waiters at 256 and frees a slot when a waiter is cancelled", () => { - const graph = makeGraph([lazy("api")]); - let state = initialState(graph); - let commandsByStep: ReadonlyArray>; - - const opens = Array.from({ length: 256 }, (_, index) => ({ - event: open("api", index + 1), - now: 0, - })); - ({ state } = run(state, opens)); - expect(state.services.get("api")?.waiters.size).toBe(256); - expect(state.services.get("api")?.phase._tag).toBe("Starting"); - - ({ state, commandsByStep } = run(state, [{ event: open("api", 257), now: 1 }])); - expect(tagsOf(commandsByStep[0])).toEqual(["FailConnection"]); - expect(state.services.get("api")?.waiters.size).toBe(256); - - ({ state, commandsByStep } = run(state, [ - { event: LifecycleEvent.WaiterCancelled({ id: "api", waiterId: 6 }), now: 2 }, - ])); - expect(commandsByStep[0]).toEqual([]); - expect(state.services.get("api")?.waiters.size).toBe(255); - // The shared launch itself is untouched by a waiter's cancellation. - expect(state.services.get("api")?.phase._tag).toBe("Starting"); - - ({ state, commandsByStep } = run(state, [{ event: open("api", 300), now: 3 }])); - expect(tagsOf(commandsByStep[0])).toEqual([]); - expect(state.services.get("api")?.waiters.size).toBe(256); - }); - it("admits a non-ready-requiring acquisition once a live session exists, bypassing target readiness", () => { const graph = makeGraph([lazy("functions")]); let state = initialState(graph); @@ -1502,40 +1411,6 @@ describe("Lifecycle reducer", () => { expect(storageAdmitted(state, "api")).toBe(true); }); - it("counts a crash whose cleanup failed once, not again when the retried cleanup confirms it", () => { - let state = runningApi(); - ({ state } = run(state, [ - { - event: LifecycleEvent.StopFailed({ - id: "api", - generation: 1, - cause: "busy", - failure: "segfault", - }), - now: 2, - }, - { event: LifecycleEvent.StopRequested({ id: "api" }), now: 3 }, - { - event: LifecycleEvent.StopFailed({ id: "api", generation: 1, cause: "busy" }), - now: 4, - }, - { event: LifecycleEvent.StopRequested({ id: "api" }), now: 5 }, - { - event: LifecycleEvent.Exited({ - id: "api", - generation: 1, - cause: undefined, - requested: true, - }), - now: 6, - }, - ])); - expect(state.services.get("api")?.breaker).toMatchObject({ - consecutiveFailures: 1, - lastCause: "segfault", - }); - }); - it("fails a restart's waiters when the stop fails, and retries the cleanup for the next waiter", () => { let state = runningApi(); let commandsByStep: ReadonlyArray>; diff --git a/packages/stack/src/Orchestrator.integration.test.ts b/packages/stack/src/Orchestrator.integration.test.ts index 852aceab20..b428597721 100644 --- a/packages/stack/src/Orchestrator.integration.test.ts +++ b/packages/stack/src/Orchestrator.integration.test.ts @@ -976,73 +976,6 @@ describe("wake and idle sleep", () => { ).pipe(Effect.provide(TestClock.layer())), ); - it.live("keeps serving traffic admitted before an idle timer fires, without a restart", () => - Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - const api = yield* makeInstance(orchestrator, "api"); - yield* orchestrator.configure({ - members: [{ id: "api", activation: "lazy", idleMillis: 1000 }], - dependencies: [], - }); - yield* orchestrator.startComposition; - yield* Effect.scoped(orchestrator.acquire("api")); - yield* TestClock.adjust("999 millis"); - - const request = yield* Scope.make(); - yield* orchestrator.acquire("api").pipe(Scope.provide(request)); - yield* TestClock.adjust("10 seconds"); - expect((yield* api.status).lifecycle).toBe("running"); - expect(yield* Ref.get(api.starts)).toHaveLength(1); - - const sleeping = yield* stopped(api).pipe(Effect.forkChild); - yield* Scope.close(request, Exit.void); - yield* TestClock.adjust("1 second"); - yield* Fiber.join(sleeping); - }), - ).pipe(Effect.provide(TestClock.layer())), - ); - - it.live( - "holds traffic that arrives after a committed sleep until the stop is confirmed, then serves it from the next launch", - () => - Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - const stopping = yield* Deferred.make(); - const finishStop = yield* Deferred.make(); - const api = yield* makeInstance(orchestrator, "api", { - stop: Deferred.succeed(stopping, undefined).pipe( - Effect.andThen(Deferred.await(finishStop)), - ), - }); - yield* orchestrator.configure({ - members: [{ id: "api", activation: "lazy", idleMillis: 1000 }], - dependencies: [], - }); - yield* orchestrator.startComposition; - yield* Effect.scoped(orchestrator.acquire("api")); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(stopping); - - const late = yield* Effect.scoped(orchestrator.acquire("api")).pipe( - Effect.forkChild({ startImmediately: true }), - ); - expect((yield* api.status).lifecycle).toBe("stopping"); - expect(yield* Ref.get(api.starts)).toHaveLength(1); - - yield* Deferred.succeed(finishStop, undefined); - yield* Fiber.join(late); - expect(yield* Ref.get(api.starts)).toHaveLength(2); - expect(yield* api.status).toMatchObject({ - lifecycle: "running", - health: "healthy", - launchId: 2, - }); - }), - ).pipe(Effect.provide(TestClock.layer())), - ); - it.live( "fails only a waiter whose wake budget expires, naming the stage, while the shared launch continues", () => @@ -1797,48 +1730,6 @@ describe("failed and crashed services", () => { }), ), ); - - it.live( - "rejects traffic beyond 256 waiters and frees a slot when a waiter is cancelled or expires", - () => - Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - const preparing = yield* Deferred.make(); - const continuePreparation = yield* Deferred.make(); - yield* makeInstance(orchestrator, "api", { - prepare: Deferred.succeed(preparing, undefined).pipe( - Effect.andThen(Deferred.await(continuePreparation)), - ), - }); - yield* orchestrator.configure({ - members: [{ id: "api", activation: "lazy" }], - dependencies: [], - }); - yield* orchestrator.startComposition; - const waitFor = Effect.scoped(orchestrator.acquire("api")).pipe( - Effect.flip, - Effect.forkChild({ startImmediately: true }), - ); - const queued = yield* Effect.forEach(Array.from({ length: 256 }), () => waitFor); - yield* Deferred.await(preparing); - expect((yield* Fiber.join(yield* waitFor)).message).toBe("api has too many waiters"); - - const [cancelled] = queued; - if (cancelled !== undefined) yield* Fiber.interrupt(cancelled); - const replacement = yield* waitFor; - yield* TestClock.adjust("120 seconds"); - expect((yield* Fiber.join(replacement)).message).toContain("wake budget exceeded"); - - const afterExpiry = yield* Effect.scoped(orchestrator.acquire("api")).pipe( - Effect.forkChild({ startImmediately: true }), - ); - yield* Deferred.succeed(continuePreparation, undefined); - yield* Fiber.join(afterExpiry); - yield* orchestrator.stopNamespace; - }), - ).pipe(Effect.provide(TestClock.layer())), - ); }); describe("wake logging", () => { diff --git a/packages/stack/src/Service.integration.test.ts b/packages/stack/src/Service.integration.test.ts index 9a492d9b42..d92c6b39fb 100644 --- a/packages/stack/src/Service.integration.test.ts +++ b/packages/stack/src/Service.integration.test.ts @@ -912,33 +912,6 @@ const startUnhealthy = (service: Standalone) => }); describe("service readiness recovery", () => { - it.live("serves readiness once a running launch recovers from a failed health check", () => - Effect.scoped( - Effect.gen(function* () { - const healthy = yield* Ref.make(false); - const { service, launches } = yield* makeProbedService(() => - Ref.get(healthy).pipe( - Effect.flatMap((ok) => (ok ? Effect.void : Effect.fail(unhealthy("still booting")))), - ), - ); - yield* startUnhealthy(service); - expect((yield* Effect.flip(service.ready)).message).toContain("still booting"); - - yield* Ref.set(healthy, true); - yield* service.ready; - - expect(yield* service.get).toMatchObject({ - lifecycle: "running", - health: "healthy", - error: undefined, - launchId: 1, - }); - expect(yield* Ref.get(launches)).toBe(1); - yield* service.stop; - }), - ), - ); - it.live("reports the initial check's failure to callers waiting on it without re-probing", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/namespace/drivers/Sqlite.integration.test.ts b/packages/stack/src/namespace/drivers/Sqlite.integration.test.ts deleted file mode 100644 index 51102a7afb..0000000000 --- a/packages/stack/src/namespace/drivers/Sqlite.integration.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Effect, Exit, FileSystem, Scope } from "effect"; -import { acquireLock, isBusy, takeLock } from "./Sqlite.ts"; - -const lockHolder = (lockPath: string, mode: "create" | "existing") => - acquireLock(lockPath, mode).pipe(Effect.flatMap(takeLock)); - -describe("SQLite lock driver", () => { - it.live("two holders in one process exclude each other until the first releases", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const directory = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-lock-" }); - const lockPath = `${directory}/holder.lock`; - - const firstScope = yield* Scope.make(); - yield* lockHolder(lockPath, "create").pipe(Scope.provide(firstScope)); - - const refused = yield* Effect.scoped(lockHolder(lockPath, "existing")).pipe(Effect.flip); - expect(isBusy(refused)).toBe(true); - - yield* Scope.close(firstScope, Exit.void); - yield* Effect.scoped(lockHolder(lockPath, "existing")); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); -}); diff --git a/packages/stack/src/port-reservation.e2e.test.ts b/packages/stack/src/port-reservation.e2e.test.ts index 12e3f10627..804ab5eaf7 100644 --- a/packages/stack/src/port-reservation.e2e.test.ts +++ b/packages/stack/src/port-reservation.e2e.test.ts @@ -11,7 +11,7 @@ import { testEngine } from "../tests/test-engine.ts"; import type { ContainerEngine } from "./runtime/Container.ts"; /** - * Step 1.4's per-user port reservation registry: a stopped stack keeps its public ports across + * The per-user port reservation registry: a stopped stack keeps its public ports across * every other stack's starts and across state roots, a restart recovers the same ports or fails * with a structured `PortConflict`, and destroy (but never stop or a kill) releases them. */ diff --git a/packages/stack/src/runtime/Container.unit.test.ts b/packages/stack/src/runtime/Container.unit.test.ts index fbd132bb2e..f0e6a5596f 100644 --- a/packages/stack/src/runtime/Container.unit.test.ts +++ b/packages/stack/src/runtime/Container.unit.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "@effect/vitest"; import { Deferred, Effect, Exit, Fiber, Ref, Scope } from "effect"; -import { ContainerError, makeHostGateway } from "./Container.ts"; +import { makeHostGateway } from "./Container.ts"; const gatewayAddress = "192.168.65.254"; @@ -15,56 +15,6 @@ const gatedProbe = Effect.fnUntraced(function* (result: string | undefined) { }); describe("host gateway", () => { - it.effect("prefetches without waiting and lets a later resolve await the same probe", () => - Effect.gen(function* () { - const gateway = yield* makeHostGateway; - const { probe, runs, release } = yield* gatedProbe(gatewayAddress); - - yield* gateway.prefetch(probe); - const waiting = yield* gateway.resolve(probe).pipe(Effect.forkChild); - yield* release; - - expect(yield* Fiber.join(waiting)).toBe(gatewayAddress); - expect(yield* gateway.resolve(probe)).toBe(gatewayAddress); - expect(yield* Ref.get(runs)).toBe(1); - }), - ); - - it.effect("retries once for a waiter whose background probe finds no target", () => - Effect.gen(function* () { - const gateway = yield* makeHostGateway; - const background = yield* gatedProbe(undefined); - const later = yield* gatedProbe(gatewayAddress); - - yield* gateway.prefetch(background.probe); - const waiting = yield* gateway - .resolve(later.probe) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* background.release; - yield* later.release; - - expect(yield* Fiber.join(waiting)).toBe(gatewayAddress); - expect(yield* Ref.get(later.runs)).toBe(1); - expect(yield* gateway.resolve(later.probe)).toBe(gatewayAddress); - expect(yield* Ref.get(later.runs)).toBe(1); - }), - ); - - it.effect("caches a probe failure for later resolves without probing again", () => - Effect.gen(function* () { - const gateway = yield* makeHostGateway; - const runs = yield* Ref.make(0); - const unsupported = new ContainerError({ operation: "host-gateway", message: "unsupported" }); - const probe = Ref.update(runs, (count) => count + 1).pipe( - Effect.andThen(Effect.fail(unsupported)), - ); - - expect(yield* Effect.flip(gateway.resolve(probe))).toBe(unsupported); - expect(yield* Effect.flip(gateway.resolve(probe))).toBe(unsupported); - expect(yield* Ref.get(runs)).toBe(1); - }), - ); - it.effect("keeps the probe running when a waiting resolve is interrupted", () => Effect.gen(function* () { const gateway = yield* makeHostGateway; diff --git a/packages/stack/src/runtime/ContainerName.unit.test.ts b/packages/stack/src/runtime/ContainerName.unit.test.ts index a663dc7a6b..a9f147dc58 100644 --- a/packages/stack/src/runtime/ContainerName.unit.test.ts +++ b/packages/stack/src/runtime/ContainerName.unit.test.ts @@ -5,26 +5,6 @@ const stackId = "0123456789abcdef0123"; const token = "a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d"; describe("identifyContainer", () => { - it("names a service container after its project and service", () => { - expect( - identifyContainer({ stackId, project: "my-app", service: "studio" }, token, false), - ).toEqual({ - name: "supabase-my-app-studio-a1b2c3d4e5f6", - composeProject: "supabase-my-app-0123456789ab", - composeService: "studio", - }); - }); - - it("marks one-shot containers with a task segment but keeps the service for grouping", () => { - const identity = identifyContainer( - { stackId, project: "my-app", service: "auth" }, - token, - true, - ); - expect(identity.name).toBe("supabase-my-app-auth-task-a1b2c3d4e5f6"); - expect(identity.composeService).toBe("auth"); - }); - it("keeps dotted folder names in the container name but not in the compose project", () => { const identity = identifyContainer( { stackId, project: "My.App", service: "rest" }, diff --git a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts index 26cdd7817a..68d4f4d663 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts @@ -278,90 +278,6 @@ describe.runIf(testEngine === "docker")( ).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); }); - it.live( - "starts a shared volume helper with the mirror image selected during preparation", - () => { - const engine = fakeHelperEngine(undefined, { volumePresent: true }); - return Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "storage-shared-helper-mirror-", - }); - const storageRoot = path.join(root, "state", "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const instanceRoot = path.join(storageRoot, "database"); - yield* fs.makeDirectory(instanceRoot, { recursive: true }); - yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const stackId = "shared-helper-mirror"; - const instanceId = "database"; - const cachePath = yield* fs.realPath(cacheRoot); - const cacheHash = yield* crypto.digest("SHA-256", new TextEncoder().encode(cachePath)); - const cacheNamespace = `cache-${Array.from(cacheHash, (byte) => - byte.toString(16).padStart(2, "0"), - ) - .join("") - .slice(0, 32)}`; - // Matches the volume name the engine target's own identity resolves, the same way - // DockerDatabaseStorage derives it: a stable digest of the state root and daemon id. - const statePath = yield* fs.realPath(path.join(root, "state")); - const stateHash = yield* crypto.digest( - "SHA-256", - new TextEncoder().encode(`${statePath}\0${dockerTarget.daemonId}`), - ); - const stateDigest = Array.from(stateHash, (byte) => byte.toString(16).padStart(2, "0")) - .join("") - .slice(0, 32); - const marker = yield* Schema.encodeEffect(Schema.fromJsonString(Marker))({ - backend: "docker", - volume: `supabase-db-${stateDigest}`, - namespace: `instance-${stackId}-${instanceId}`, - cacheNamespace, - daemonId: dockerTarget.daemonId, - initialized: false, - }); - yield* fs.writeFileString( - path.join(instanceRoot, ".supabase-database-storage.json"), - marker, - ); - const container = yield* makeContainerRuntime({ - target: dockerTarget, - root, - imageMirrors: () => [helperMirror], - }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const helpers = yield* makeDockerHelperRegistry("mirror-test"); - const storage = yield* makeDockerDatabaseStorage({ - runtime: "docker", - target: dockerTarget, - stackId, - instanceId, - instanceRoot, - root: storageRoot, - cacheRoot, - fs, - path, - crypto, - container, - spawner, - helpers, - }); - - yield* storage.prepare("17"); - - const run = engine.commands.find((args) => args[0] === "run"); - const shellIndex = run?.indexOf("/bin/sh") ?? -1; - expect(run?.[shellIndex - 1]).toBe(helperMirror); - expect( - engine.commands.filter((args) => args[0] === "pull").map((args) => args.at(-1)), - ).toEqual([expect.stringContaining("ghcr.io/supabase/cli/postgres:"), helperMirror]); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); - }, - ); - it.live("round-trips PostgreSQL 15 data with its catalog image", () => Effect.scoped( Effect.gen(function* () { From dd637b8b00d56c4e6613921069a362b8f71770a5 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 18:47:24 +0200 Subject: [PATCH 106/142] refactor(cli): derive stack port env overrides from the config path --- apps/cli/src/command-internal/stack-config.ts | 182 +++++------------- 1 file changed, 44 insertions(+), 138 deletions(-) diff --git a/apps/cli/src/command-internal/stack-config.ts b/apps/cli/src/command-internal/stack-config.ts index b5acd04dcf..cb151f385f 100644 --- a/apps/cli/src/command-internal/stack-config.ts +++ b/apps/cli/src/command-internal/stack-config.ts @@ -127,120 +127,26 @@ const section = (document: Readonly> | undefined, name: return isRecord(value) ? value : undefined; }; -const explicitPort = ( +/** The configured port when `path` is set in `config.toml` or by its `SUPABASE_` env var, else undefined. */ +const portIfSet = ( document: Readonly> | undefined, - sectionName: string, - key: string, -): number | undefined => { - const value = section(document, sectionName)?.[key]; - return typeof value === "number" ? value : undefined; -}; - -const envPortOrConfigured = ( - name: string, - document: Readonly> | undefined, - sectionName: string, - key: string, - configured: number, env: Readonly>, -): number | undefined => { - if (envOverride(name, undefined, env) !== undefined) return configured; - return explicitPort(document, sectionName, key) === undefined ? undefined : configured; -}; - -const envNestedPortOrConfigured = ( - name: string, - document: Readonly> | undefined, - sectionName: string, - nestedSection: string, - key: string, + path: readonly [string, ...string[]], configured: number, - env: Readonly>, ): number | undefined => { - if (envOverride(name, undefined, env) !== undefined) return configured; - const nested = section(section(document, sectionName), nestedSection); - return typeof nested?.[key] === "number" ? configured : undefined; -}; - -/** A port setting's `config.toml` section/key and the `SUPABASE_*` env var that overrides it. */ -interface PortSetting { - readonly envVar: string; - readonly section: string; - readonly nestedSection?: string; - readonly key: string; -} - -const DB_PORT: PortSetting = { - envVar: "SUPABASE_DB_PORT", - section: "db", - key: "port", -}; -const API_PORT: PortSetting = { - envVar: "SUPABASE_API_PORT", - section: "api", - key: "port", -}; -const STUDIO_PORT: PortSetting = { - envVar: "SUPABASE_STUDIO_PORT", - section: "studio", - key: "port", -}; -const DB_POOLER_PORT: PortSetting = { - envVar: "SUPABASE_DB_POOLER_PORT", - section: "db", - nestedSection: "pooler", - key: "port", -}; -const LOCAL_SMTP_PORT: PortSetting = { - envVar: "SUPABASE_LOCAL_SMTP_PORT", - section: "local_smtp", - key: "port", -}; -const LOCAL_SMTP_SMTP_PORT: PortSetting = { - envVar: "SUPABASE_LOCAL_SMTP_SMTP_PORT", - section: "local_smtp", - key: "smtp_port", -}; -const LOCAL_SMTP_POP3_PORT: PortSetting = { - envVar: "SUPABASE_LOCAL_SMTP_POP3_PORT", - section: "local_smtp", - key: "pop3_port", -}; -const ANALYTICS_PORT: PortSetting = { - envVar: "SUPABASE_ANALYTICS_PORT", - section: "analytics", - key: "port", -}; -const ANALYTICS_VECTOR_PORT: PortSetting = { - envVar: "SUPABASE_ANALYTICS_VECTOR_PORT", - section: "analytics", - key: "vector_port", -}; -const EDGE_RUNTIME_INSPECTOR_PORT: PortSetting = { - envVar: "SUPABASE_EDGE_RUNTIME_INSPECTOR_PORT", - section: "edge_runtime", - key: "inspector_port", + if (envOverride(["SUPABASE", ...path].join("_").toUpperCase(), undefined, env) !== undefined) { + return configured; + } + const key = path[path.length - 1] ?? ""; + const parent = path + .slice(0, -1) + .reduce> | undefined>( + (node, name) => section(node, name), + document, + ); + return typeof parent?.[key] === "number" ? configured : undefined; }; -/** Resolves one `PortSetting` against the loaded document and env, picking the nested variant when needed. */ -const resolvePort = ( - setting: PortSetting, - document: Readonly> | undefined, - configured: number, - env: Readonly>, -): number | undefined => - setting.nestedSection === undefined - ? envPortOrConfigured(setting.envVar, document, setting.section, setting.key, configured, env) - : envNestedPortOrConfigured( - setting.envVar, - document, - setting.section, - setting.nestedSection, - setting.key, - configured, - env, - ); - const authProviderNames = [ "apple", "azure", @@ -1149,53 +1055,53 @@ export const loadStackConfig = Effect.fn("StackConfig.load")( }, catch: (cause) => new StackConfigError({ message: String(cause) }), }); - const dbPort = resolvePort( - DB_PORT, + const dbPort = portIfSet( document, - validatedConfig.db.port, context.projectEnvValues, + ["db", "port"], + validatedConfig.db.port, ); - const apiPort = resolvePort( - API_PORT, + const apiPort = portIfSet( document, - validatedConfig.api.port, context.projectEnvValues, + ["api", "port"], + validatedConfig.api.port, ); - const studioPort = resolvePort( - STUDIO_PORT, + const studioPort = portIfSet( document, - validatedConfig.studio.port, context.projectEnvValues, + ["studio", "port"], + validatedConfig.studio.port, ); - const poolerPort = resolvePort( - DB_POOLER_PORT, + const poolerPort = portIfSet( document, - validatedConfig.db.pooler.port, context.projectEnvValues, + ["db", "pooler", "port"], + validatedConfig.db.pooler.port, ); - const mailPort = resolvePort( - LOCAL_SMTP_PORT, + const mailPort = portIfSet( document, - validatedConfig.local_smtp.port, context.projectEnvValues, + ["local_smtp", "port"], + validatedConfig.local_smtp.port, ); - const mailSmtpPort = resolvePort( - LOCAL_SMTP_SMTP_PORT, + const mailSmtpPort = portIfSet( document, - validatedConfig.local_smtp.smtp_port ?? 0, context.projectEnvValues, + ["local_smtp", "smtp_port"], + validatedConfig.local_smtp.smtp_port ?? 0, ); - const mailPop3Port = resolvePort( - LOCAL_SMTP_POP3_PORT, + const mailPop3Port = portIfSet( document, - validatedConfig.local_smtp.pop3_port ?? 0, context.projectEnvValues, + ["local_smtp", "pop3_port"], + validatedConfig.local_smtp.pop3_port ?? 0, ); - const analyticsPort = resolvePort( - ANALYTICS_PORT, + const analyticsPort = portIfSet( document, - validatedConfig.analytics.port, context.projectEnvValues, + ["analytics", "port"], + validatedConfig.analytics.port, ); const poolMode = validatedConfig.db.pooler.pool_mode === "session" ? ("session" as const) : "transaction"; @@ -1325,11 +1231,11 @@ export const loadStackConfig = Effect.fn("StackConfig.load")( config: { apiKey: "api-key" }, endpoints: { http: endpoint( - resolvePort( - ANALYTICS_VECTOR_PORT, + portIfSet( document, - validatedConfig.analytics.vector_port ?? 0, context.projectEnvValues, + ["analytics", "vector_port"], + validatedConfig.analytics.vector_port ?? 0, ), ), }, @@ -1361,11 +1267,11 @@ export const loadStackConfig = Effect.fn("StackConfig.load")( endpoints: { http: endpoint(apiPort), inspector: endpoint( - resolvePort( - EDGE_RUNTIME_INSPECTOR_PORT, + portIfSet( document, - validatedConfig.edge_runtime.inspector_port, context.projectEnvValues, + ["edge_runtime", "inspector_port"], + validatedConfig.edge_runtime.inspector_port, ), ), }, From df65d945ec1f7410904b9f2502bfc938085197dd Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 18:47:25 +0200 Subject: [PATCH 107/142] refactor(cli): use snake_case keys in stack stop and destroy JSON output --- .../src/commands/experimental/stack/destroy/SIDE_EFFECTS.md | 2 +- .../src/commands/experimental/stack/destroy/destroy.handler.ts | 3 ++- .../destroy/destroy.runtime-unavailable.integration.test.ts | 2 +- apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md | 2 +- apps/cli/src/commands/experimental/stack/stop/stop.handler.ts | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md index 3aa9e93e06..e602f43514 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md @@ -64,7 +64,7 @@ remains the backstop when no owner is left alive to do so. Text prints `Stack destroyed.`, or, when engine cleanup was skipped, `Stack could not be fully destroyed because is unreachable; restore it and run "supabase stack destroy --stack-id " again.` JSON and stream-json success data contain `destroyed` (`false` when engine -cleanup was skipped), `id`, and `runtimeCleanup` (`complete` or `skipped`); a +cleanup was skipped), `id`, and `runtime_cleanup` (`complete` or `skipped`); a skipped cleanup also carries `engine`. Exit 0 on destruction or skipped engine cleanup, 1 on invalid flags, missing selection, rejected/cancelled confirmation or cleanup failure, and 130 on interruption. Standard command telemetry is unchanged, with no custom events. diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts index 2d3ffb13c2..ef2f20cde5 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts @@ -114,7 +114,8 @@ export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( yield* output.success("", { destroyed: result.runtimeCleanup === "complete", id: target.id, - ...result, + runtime_cleanup: result.runtimeCleanup, + ...(result.runtimeCleanup === "skipped" ? { engine: result.engine } : {}), }); else if (result.runtimeCleanup === "complete") yield* output.raw(`Stack ${target.id} destroyed.\n`); diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts index 7185076daa..691e822f8a 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts @@ -91,7 +91,7 @@ it.live("reports skipped engine cleanup in the JSON result", () => data: { destroyed: false, id: f.stack.id, - runtimeCleanup: "skipped", + runtime_cleanup: "skipped", engine: "docker", }, }), diff --git a/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md index aff47bcbc3..da9a225994 100644 --- a/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md @@ -25,7 +25,7 @@ service during shutdown is refused. Shutdown is one-way: a failed stop leaves the stack registered and the owner exits, and the next stop retries. Text confirms each successful shutdown and identifies each stack that had no owner. -JSON and stream-json success data contain `stopped` and `notRunning` ID arrays. +JSON and stream-json success data contain `stopped` and `not_running` ID arrays. A missing default selection reports `found: false`; an unknown explicit name or ID fails. A single selection reads only the selected stack's state document; an unreadable document fails the selection instead of being reported as missing. diff --git a/apps/cli/src/commands/experimental/stack/stop/stop.handler.ts b/apps/cli/src/commands/experimental/stack/stop/stop.handler.ts index df8858cb7a..38573871d8 100644 --- a/apps/cli/src/commands/experimental/stack/stop/stop.handler.ts +++ b/apps/cli/src/commands/experimental/stack/stop/stop.handler.ts @@ -154,6 +154,6 @@ export const stackStop = Effect.fn("experimental.stack.stop")(function* (flags: for (const id of notRunning) yield* output.raw(`Stack ${id} was not running; leftover resources were reclaimed.\n`); if (results.length === 0) yield* output.raw("No managed stacks found.\n"); - } else yield* output.success("", { stopped, notRunning }); + } else yield* output.success("", { stopped, not_running: notRunning }); }).pipe(Effect.ensuring(telemetry.flush)); }); From 7752e03f66c92de65dbe535c2b60e83c337c11df Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 18:47:26 +0200 Subject: [PATCH 108/142] test(cli): fold duplicated stack start and runtime tests into their integration suites --- .../stack-runtime.unit.test.ts | 26 -- .../stack/prepare/prepare.integration.test.ts | 14 +- .../stack/start/start.e2e.test.ts | 47 --- .../stack/start/start.integration.test.ts | 298 ++++++++---------- 4 files changed, 153 insertions(+), 232 deletions(-) delete mode 100644 apps/cli/src/command-internal/stack-runtime.unit.test.ts diff --git a/apps/cli/src/command-internal/stack-runtime.unit.test.ts b/apps/cli/src/command-internal/stack-runtime.unit.test.ts deleted file mode 100644 index 79794d8306..0000000000 --- a/apps/cli/src/command-internal/stack-runtime.unit.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { BunServices } from "@effect/platform-bun"; -import { describe, expect, it } from "@effect/vitest"; -import { Effect, Layer, Ref } from "effect"; -import { runtimeInfoLayer } from "../shared/runtime/runtime-info.layer.ts"; -import { - CommandTelemetryAttributes, - type CommandTelemetryAttributeValues, -} from "../telemetry/command-telemetry-attributes.ts"; -import { selectStackRuntime } from "./stack-runtime.ts"; - -describe("selectStackRuntime", () => { - it.effect("records the selected runtime on the command event", () => - Effect.gen(function* () { - const recorded = yield* Ref.make({}); - - const runtime = yield* selectStackRuntime("podman").pipe( - Effect.provideService(CommandTelemetryAttributes, { - record: (values) => Ref.update(recorded, (current) => ({ ...current, ...values })), - }), - ); - - expect(runtime).toBe("podman"); - expect((yield* Ref.get(recorded)).stack_runtime).toBe("podman"); - }).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, runtimeInfoLayer))), - ); -}); diff --git a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts index e89295c1dc..565e59b568 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts @@ -22,6 +22,7 @@ import { mockTelemetryStateTracked, } from "../../../../../tests/helpers/command-mocks.ts"; import { mockOutput, mockRuntimeInfo } from "../../../../../tests/helpers/mocks.ts"; +import { CommandTelemetryAttributes } from "../../../../telemetry/command-telemetry-attributes.ts"; import { StackApi, StackTargetResolver } from "../stack.shared.ts"; import { stackPrepare } from "./prepare.handler.ts"; import type { StackPrepareFlags } from "./prepare.command.ts"; @@ -62,6 +63,7 @@ const makeFixture = (root: string, options: FixtureOptions = {}) => { options.engines ?? { docker: "missing", podman: "missing" }, ); const createdRuntimes: Array = []; + const recordedRuntimes: Array = []; let openCount = 0; let prepareCount = 0; let startCount = 0; @@ -167,6 +169,12 @@ const makeFixture = (root: string, options: FixtureOptions = {}) => { mockCommandSettings({ workdir: root, supabaseHome: root }), output.layer, telemetry.layer, + Layer.succeed(CommandTelemetryAttributes, { + record: (values) => + Effect.sync(() => { + if (values.stack_runtime !== undefined) recordedRuntimes.push(values.stack_runtime); + }), + }), Layer.succeed(StackTargetResolver, { resolve: (input) => Effect.succeed({ @@ -199,6 +207,9 @@ const makeFixture = (root: string, options: FixtureOptions = {}) => { get createdRuntimes() { return createdRuntimes; }, + get recordedRuntimes() { + return recordedRuntimes; + }, get openCount() { return openCount; }, @@ -441,7 +452,7 @@ describe("stack prepare automatic runtime selection", () => { ), ); - it.live("honors an explicit runtime without probing any engine", () => + it.live("honors an explicit runtime without probing any engine and records it on telemetry", () => makeProject(databaseOnlyConfig).pipe( Effect.flatMap((root) => { const fixture = makeFixture(root, { engines: { docker: "running", podman: "running" } }); @@ -450,6 +461,7 @@ describe("stack prepare automatic runtime selection", () => { Effect.tap(() => Effect.sync(() => { expect(fixture.createdRuntimes).toEqual(["podman"]); + expect(fixture.recordedRuntimes).toEqual(["podman"]); expect(fixture.probes).toEqual([]); expect(fixture.runtimeNotices).toEqual([]); }), diff --git a/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts b/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts index 616b4f6f11..67304af651 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts @@ -92,7 +92,6 @@ const FollowEventSchema = Schema.Struct({ const VariablesSchema = Schema.Record(Schema.String, Schema.String); const StartResultSchema = Schema.Struct({ id: Schema.String }); -const StackListSchema = Schema.Struct({ stacks: Schema.Array(Schema.Unknown) }); const minimalConfig = `project_id = "compiled-stack-start-e2e" @@ -560,50 +559,4 @@ describe("stack start (compiled e2e)", () => { }), ), ); - - test( - "removes a stack registration that fails to start because Docker is unreachable", - { timeout: CLEANUP_TIMEOUT_MS }, - () => - runNode( - Effect.gen(function* () { - home = makeTempHome(); - projectDir = yield* makeTempDirectory("/tmp/supabase-stack-start-docker-down-e2e-"); - yield* makeDirectory(join(projectDir, "supabase"), { recursive: true }); - yield* writeText(join(projectDir, "supabase", "config.toml"), minimalConfig); - // A `docker` first on PATH that reports an unreachable daemon, as the real CLI does. - const binDir = join(projectDir, "bin"); - yield* makeDirectory(binDir); - yield* writeText( - join(binDir, "docker"), - "#!/bin/sh\necho 'Cannot connect to the Docker daemon at unix:///shim/docker.sock. Is the docker daemon running?' >&2\nexit 1\n", - ); - yield* withFs((fs) => fs.chmod(join(binDir, "docker"), 0o755)); - - const result = yield* runSupabaseEffect(["stack", "start", "--runtime", "docker"], { - cwd: projectDir, - home: home.dir, - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the CLI subprocess resolves `docker` from PATH. - env: { PATH: `${binDir}:${process.env.PATH ?? ""}` }, - exitTimeoutMs: CLEANUP_TIMEOUT_MS, - }); - expect(result.exitCode, `stdout:\n${result.stdout}\nstderr:\n${result.stderr}`).not.toBe( - 0, - ); - expect(result.stderr).toContain("Cannot connect to the Docker daemon"); - expect(result.stderr).not.toContain("Failed to stop stack host"); - - const list = yield* runSupabaseEffect(["stack", "list", "--output-format", "json"], { - home: home.dir, - env: { SUPABASE_EXPERIMENTAL_STACK: "1" }, - exitTimeoutMs: CLEANUP_TIMEOUT_MS, - }); - expect(list.exitCode, `stdout:\n${list.stdout}\nstderr:\n${list.stderr}`).toBe(0); - const listed = yield* Schema.decodeEffect(Schema.fromJsonString(StackListSchema))( - list.stdout.trim(), - ); - expect(listed.stacks).toEqual([]); - }), - ), - ); }); diff --git a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts index 9da7a7a7c6..e3adbca6b7 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts @@ -86,11 +86,7 @@ const session: DbSession = { const instance = ( creation: ServiceCreation, id: string, - lifecycle: () => "stopped" | "starting" | "running", - wakeEnabled: () => boolean, - health: () => "starting" | "healthy" | "unhealthy" | undefined = () => - lifecycle() === "running" ? "healthy" : undefined, - ready: () => Effect.Effect = () => Effect.void, + view: () => MemberView, ): ServiceInstances[ServiceCreation["service"]] => { const status = (config: ServiceCreation) => ({ id, @@ -108,20 +104,20 @@ const instance = ( ] : [], config, - lifecycle: lifecycle(), - health: health(), + lifecycle: view().lifecycle, + health: view().health, error: undefined, cleanupError: undefined, exit: undefined, currentOperation: undefined, launchId: undefined, - wakeEnabled: wakeEnabled(), + wakeEnabled: view().wakeEnabled, registered: true, }); const base = { id, start: Effect.void, - ready: Effect.suspend(ready), + ready: Effect.suspend(() => view().ready), stop: Effect.void, restart: () => Effect.void, destroy: Effect.void, @@ -226,12 +222,45 @@ const requireConcreteCreation = (creation: ServiceCreationInput): ServiceCreatio }; }; +interface MemberView { + readonly lifecycle: "stopped" | "starting" | "running"; + readonly wakeEnabled: boolean; + readonly health: "starting" | "healthy" | "unhealthy" | undefined; + readonly ready: Effect.Effect; +} + interface MemberStatus { readonly lifecycle?: "stopped" | "starting" | "running"; readonly wakeEnabled?: boolean; readonly health?: "starting" | "healthy" | "unhealthy"; } +const credentialsFor = ( + database: ServiceCreationInput | undefined, + options: Parameters[1], +): StackCredentials => { + const config = database?.service === "database" ? database.config : undefined; + return { + jwtSecret: + config?.jwtSecret === undefined ? DEFAULT_LOCAL_JWT_SECRET : Redacted.value(config.jwtSecret), + postgresRootKey: + config?.rootKey === undefined ? DEFAULT_POSTGRES_ROOT_KEY : Redacted.value(config.rootKey), + databasePassword: + config?.databasePassword === undefined + ? DEFAULT_LOCAL_DATABASE_PASSWORD + : Redacted.value(config.databasePassword), + publishableKey: options?.keys?.publishableKey ?? "sb_publishable_test", + secretKey: options?.keys?.secretKey ?? "sb_secret_test", + anonKey: options?.keys?.anonKey ?? "anon-token", + serviceRoleKey: options?.keys?.serviceRoleKey ?? "service-token", + jwks: '{"keys":[]}', + gotrueJwtKeys: options?.keys?.gotrueJwtKeys ?? "[]", + remoteJwks: options?.keys?.remoteJwks ?? "[]", + anonKeyIsOverride: options?.keys?.anonKeyIsOverride ?? false, + serviceRoleKeyIsOverride: options?.keys?.serviceRoleKeyIsOverride ?? false, + }; +}; + const fakeStack = (compositionStart?: Stack["composition"]["start"]) => { let members: Array = []; let stopped = 0; @@ -244,19 +273,17 @@ const fakeStack = (compositionStart?: Stack["composition"]["start"]) => { let activations = new Map(); const memberStatuses = new Map(); const memberReadiness = new Map>(); - let savedCredentials: StackCredentials = { - jwtSecret: DEFAULT_LOCAL_JWT_SECRET, - postgresRootKey: DEFAULT_POSTGRES_ROOT_KEY, - databasePassword: DEFAULT_LOCAL_DATABASE_PASSWORD, - publishableKey: "sb_publishable_test", - secretKey: "sb_secret_test", - anonKey: "anon-token", - serviceRoleKey: "service-token", - jwks: '{"keys":[]}', - gotrueJwtKeys: "[]", - remoteJwks: "[]", - anonKeyIsOverride: false, - serviceRoleKeyIsOverride: false, + let savedCredentials = credentialsFor(undefined, undefined); + const memberView = (id: string): MemberView => { + const status = memberStatuses.get(id); + const memberLifecycle = status?.lifecycle ?? lifecycle; + return { + lifecycle: memberLifecycle, + wakeEnabled: + status?.wakeEnabled ?? (lifecycle === "running" && activations.get(id) === "lazy"), + health: status?.health ?? (memberLifecycle === "running" ? "healthy" : undefined), + ready: memberReadiness.get(id) ?? Effect.void, + }; }; const stack: Stack = { id: "a".repeat(64), @@ -281,31 +308,10 @@ const fakeStack = (compositionStart?: Stack["composition"]["start"]) => { supabase: (creations: ReadonlyArray, options) => Effect.sync(() => { composed += 1; - const database = creations.find((creation) => creation.service === "database"); - const jwtSecret = - database?.service === "database" && database.config.jwtSecret !== undefined - ? Redacted.value(database.config.jwtSecret) - : DEFAULT_LOCAL_JWT_SECRET; - savedCredentials = { - jwtSecret, - postgresRootKey: - database?.service === "database" && database.config.rootKey !== undefined - ? Redacted.value(database.config.rootKey) - : DEFAULT_POSTGRES_ROOT_KEY, - databasePassword: - database?.service === "database" && database.config.databasePassword !== undefined - ? Redacted.value(database.config.databasePassword) - : DEFAULT_LOCAL_DATABASE_PASSWORD, - publishableKey: options?.keys?.publishableKey ?? "sb_publishable_test", - secretKey: options?.keys?.secretKey ?? "sb_secret_test", - anonKey: options?.keys?.anonKey ?? "anon-token", - serviceRoleKey: options?.keys?.serviceRoleKey ?? "service-token", - jwks: '{"keys":[]}', - gotrueJwtKeys: options?.keys?.gotrueJwtKeys ?? "[]", - remoteJwks: options?.keys?.remoteJwks ?? "[]", - anonKeyIsOverride: options?.keys?.anonKeyIsOverride ?? false, - serviceRoleKeyIsOverride: options?.keys?.serviceRoleKeyIsOverride ?? false, - }; + savedCredentials = credentialsFor( + creations.find((creation) => creation.service === "database"), + options, + ); const previousMembers = members; members = creations.map((creation) => { const previous = previousMembers.find(({ service }) => service === creation.service); @@ -313,20 +319,7 @@ const fakeStack = (compositionStart?: Stack["composition"]["start"]) => { previous !== undefined && options?.reuseIds?.includes(previous.id) ? previous.id : `${creation.service}-member-${composed}`; - return instance( - requireConcreteCreation(creation), - id, - () => memberStatuses.get(id)?.lifecycle ?? lifecycle, - () => - memberStatuses.get(id)?.wakeEnabled ?? - (lifecycle === "running" && activations.get(id) === "lazy"), - () => { - const status = memberStatuses.get(id); - if (status?.health !== undefined) return status.health; - return (status?.lifecycle ?? lifecycle) === "running" ? "healthy" : undefined; - }, - () => memberReadiness.get(id) ?? Effect.void, - ); + return instance(requireConcreteCreation(creation), id, () => memberView(id)); }); activations = new Map( members.map(({ id, service }) => [ @@ -1441,108 +1434,97 @@ describe("experimental stack start", () => { }).pipe(Effect.provide(BunServices.layer)), ); - // The `#!/bin/sh` shim is never picked up on Windows, which only resolves `docker.exe` on PATH. - it.live.skipIf(process.platform === "win32")( - "leaves no stack registered when a new stack's owner cannot reach the Docker daemon", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-create-failure-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "create-failure"\n', - ); - yield* fs.makeDirectory(`${root}/bin`); - yield* fs.writeFileString( - `${root}/bin/docker`, - "#!/bin/sh\necho 'Cannot connect to the Docker daemon at unix:///shim/docker.sock. Is the docker daemon running?' >&2\nexit 1\n", - ); - yield* fs.chmod(`${root}/bin/docker`, 0o755); - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the detached host subprocess inherits PATH; this is not application config. - const originalPath = process.env.PATH; - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. - process.env.PATH = `${root}/bin:${originalPath ?? ""}`; - yield* Effect.addFinalizer(() => - Effect.sync(() => { - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. - process.env.PATH = originalPath; - }), - ); - const output = mockOutput(); - const target = Layer.succeed(StackTargetResolver, { - resolve: () => - Effect.succeed({ projectRoot: root, runtime: "docker" as const, hostRunning: false }), - }); - const api = stackApiLayer.pipe(Layer.provide(BunServices.layer)); - - const error = yield* stackStart(flags()).pipe( - Effect.flip, - Effect.provide(Layer.mergeAll(layers(root, fakeStack(), output, false), target, api)), - ); - expect(error.message).toContain("Cannot connect to the Docker daemon"); - expect(error).toBeInstanceOf(StackCommandStartError); - if (error instanceof StackCommandStartError) { - expect(error.reason).toBe("runtime"); - expect(error.suggestion).toContain("Docker CLI or daemon isn't reachable"); - } - expect(output.stderrText).not.toContain("Failed to stop"); - - const stacks = yield* StackApi.pipe( - Effect.flatMap((stackApi) => - stackApi.discover({ stateRoot: `${root}/.supabase/stacks` }), - ), - Effect.provide(api), - ); - expect(stacks).toEqual([]); - }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), - ); - - it.live("leaves no stack registered when a new stack's owner finds no Docker CLI", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-create-failure-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "create-failure"\n'); + const unreachableDocker = [ + { + name: "cannot reach the Docker daemon", + // The `#!/bin/sh` shim is never picked up on Windows, which only resolves `docker.exe` on PATH. + posixOnly: true, + message: "Cannot connect to the Docker daemon", + // A `docker` first on PATH that reports an unreachable daemon, as the real CLI does. + path: (root: string, inherited: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + yield* fs.makeDirectory(`${root}/bin`); + yield* fs.writeFileString( + `${root}/bin/docker`, + "#!/bin/sh\necho 'Cannot connect to the Docker daemon at unix:///shim/docker.sock. Is the docker daemon running?' >&2\nexit 1\n", + ); + yield* fs.chmod(`${root}/bin/docker`, 0o755); + return `${root}/bin:${inherited}`; + }), + }, + { + name: "finds no Docker CLI", + posixOnly: false, + message: "docker context show", // An empty PATH hides any installed Docker CLI on every platform. - yield* fs.makeDirectory(`${root}/empty-bin`); - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- Windows names the variable `Path`; the detached owner inherits it. - const envKeys = Object.keys(process.env); - const pathKey = envKeys.find((key) => key.toUpperCase() === "PATH") ?? "PATH"; - yield* withEnvVar( - pathKey, - `${root}/empty-bin`, + path: (root: string) => Effect.gen(function* () { - const output = mockOutput(); - const target = Layer.succeed(StackTargetResolver, { - resolve: () => - Effect.succeed({ projectRoot: root, runtime: "docker" as const, hostRunning: false }), - }); - const api = stackApiLayer.pipe(Layer.provide(BunServices.layer)); + const fs = yield* FileSystem.FileSystem; + yield* fs.makeDirectory(`${root}/empty-bin`); + return `${root}/empty-bin`; + }), + }, + ]; - const error = yield* stackStart(flags()).pipe( - Effect.flip, - Effect.provide(Layer.mergeAll(layers(root, fakeStack(), output, false), target, api)), + for (const row of unreachableDocker) { + it.live.skipIf(row.posixOnly && process.platform === "win32")( + `leaves no stack registered when a new stack's owner ${row.name}`, + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-create-failure-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "create-failure"\n', ); - expect(error.message).toContain("docker context show"); - expect(error).toBeInstanceOf(StackCommandStartError); - if (error instanceof StackCommandStartError) { - expect(error.reason).toBe("runtime"); - expect(error.suggestion).toContain("Docker CLI or daemon isn't reachable"); - } - expect(output.stderrText).not.toContain("Failed to stop"); - - const stacks = yield* StackApi.pipe( - Effect.flatMap((stackApi) => - stackApi.discover({ stateRoot: `${root}/.supabase/stacks` }), - ), - Effect.provide(api), + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- Windows names the variable `Path`; the detached owner inherits it. + const envKeys = Object.keys(process.env); + const pathKey = envKeys.find((key) => key.toUpperCase() === "PATH") ?? "PATH"; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the detached host subprocess inherits PATH; this is not application config. + const path = yield* row.path(root, process.env[pathKey] ?? ""); + yield* withEnvVar( + pathKey, + path, + Effect.gen(function* () { + const output = mockOutput(); + const target = Layer.succeed(StackTargetResolver, { + resolve: () => + Effect.succeed({ + projectRoot: root, + runtime: "docker" as const, + hostRunning: false, + }), + }); + const api = stackApiLayer.pipe(Layer.provide(BunServices.layer)); + + const error = yield* stackStart(flags()).pipe( + Effect.flip, + Effect.provide( + Layer.mergeAll(layers(root, fakeStack(), output, false), target, api), + ), + ); + expect(error.message).toContain(row.message); + expect(error).toBeInstanceOf(StackCommandStartError); + if (error instanceof StackCommandStartError) { + expect(error.reason).toBe("runtime"); + expect(error.suggestion).toContain("Docker CLI or daemon isn't reachable"); + } + expect(output.stderrText).not.toContain("Failed to stop"); + + const stacks = yield* StackApi.pipe( + Effect.flatMap((stackApi) => + stackApi.discover({ stateRoot: `${root}/.supabase/stacks` }), + ), + Effect.provide(api), + ); + expect(stacks).toEqual([]); + }), ); - expect(stacks).toEqual([]); - }), - ); - }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), - ); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + } it.live( "stops, without destroying, the owner when startup fails after the stack is acquired", From ff7c6375d9d4d1c180c95f7c7967cdda71f41b07 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 18:57:32 +0200 Subject: [PATCH 109/142] fix(stack): stop routing to a crashed workload while its cleanup runs --- packages/stack/ARCHITECTURE.md | 12 +- packages/stack/src/Lifecycle.ts | 41 +++- packages/stack/src/Lifecycle.unit.test.ts | 175 ++++++++++++++---- .../src/Orchestrator.integration.test.ts | 48 +++++ packages/stack/src/Service.ts | 9 +- packages/stack/tests/whole-stack/idle.ts | 21 ++- packages/stack/tests/whole-stack/workloads.ts | 30 +++ 7 files changed, 282 insertions(+), 54 deletions(-) diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index ad2122c159..9ad8df2d6b 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -80,10 +80,10 @@ stateDiagram-v2 Stopped --> Starting: demand or explicit start Starting --> Running: session available Starting --> Failed: launch failed, cleanup confirmed - Running --> Failed: unexpected exit, cleanup confirmed - Running --> Stopping: stop, restart or idle sleep - Starting --> Stopping: stop or restart - Stopping --> Stopped: cleanup confirmed + Running --> Stopping: unexpected exit, stop, restart or idle sleep + Starting --> Stopping: unexpected exit, stop or restart + Stopping --> Failed: unexpected exit, cleanup confirmed + Stopping --> Stopped: stop, restart or idle sleep, cleanup confirmed Stopping --> Stopping: cleanup failed, retried on request Failed --> Starting: demand, behind the breaker Failed --> Stopped: explicit stop @@ -98,7 +98,7 @@ stateDiagram-v2 end note ``` -Each phase after `Stopped` belongs to one generation. The final marker means the registration no longer exists. `destroy` of a running instance stops it, then removes its data under a storage reservation, which it takes once any running storage operation releases it. A generation whose cleanup fails stays stopping, holding its resources; the next stop, restart, start or waiter retries the remaining cleanup. A dead process is never reported as running, nor unfinished cleanup as stopped. Failed data removal leaves the registration present. Restart composes stop and launch; it adds no stable state. +Each phase after `Stopped` belongs to one generation. The final marker means the registration no longer exists. `destroy` of a running instance stops it, then removes its data under a storage reservation, which it takes once any running storage operation releases it. A generation whose cleanup fails stays stopping, holding its resources; the next stop, restart, start or waiter retries the remaining cleanup. A workload that exits on its own enters `Stopping` at once, so no traffic routes to its dead endpoint while cleanup runs; it counts toward the breaker then, and ends `Failed` once cleanup is confirmed. A dead process is never reported as running, nor unfinished cleanup as stopped. Failed data removal leaves the registration present. Restart composes stop and launch; it adds no stable state. ### Chart 2: health does not hold the lifecycle gate @@ -247,7 +247,7 @@ Explicit stop and restart wait on the reducer state itself: a stop completes whe ### The execution boundary -`Service.ts` executes one generation at a time and owns its resources from before preparation until cleanup is confirmed. A launch prepares, acquires the runtime under the service's execution lock, reports the live session, then runs the readiness check. A stop fences its generation under the same lock, so a launch of that generation that has not acquired its runtime never will; it then interrupts the attempt while it prepares or checks readiness, and cleans only that generation's session, never a newer one. Every outcome is reported: `SessionAvailable`, `LaunchSucceeded`, `ReadinessLost`/`ReadinessRecovered`, `LaunchFailed` and `Exited` once cleanup is confirmed, or `StopFailed` while resources are still held. +`Service.ts` executes one generation at a time and owns its resources from before preparation until cleanup is confirmed. A launch prepares, acquires the runtime under the service's execution lock, reports the live session, then runs the readiness check. A stop fences its generation under the same lock, so a launch of that generation that has not acquired its runtime never will; it then interrupts the attempt while it prepares or checks readiness, and cleans only that generation's session, never a newer one. Every outcome is reported: `SessionAvailable`, `LaunchSucceeded`, `ReadinessLost`/`ReadinessRecovered`, `LaunchFailed`, `SessionLost` as soon as a runtime exits on its own, and `Exited` once cleanup is confirmed, or `StopFailed` while resources are still held. A generation whose cleanup failed stays `stopping` with its cleanup failure recorded. Nothing launches and no storage operation is admitted while it holds resources, and its waiters fail with the cleanup error. The next explicit stop, restart or start, or the next waiter, retries only the remaining cleanup. diff --git a/packages/stack/src/Lifecycle.ts b/packages/stack/src/Lifecycle.ts index 1d8c05460c..5ba9ece430 100644 --- a/packages/stack/src/Lifecycle.ts +++ b/packages/stack/src/Lifecycle.ts @@ -86,7 +86,8 @@ type Phase = Data.TaggedEnum<{ Stopped: {}; Starting: { readonly generation: number; readonly stage: Stage }; Running: { readonly generation: number; readonly ready: boolean }; - Stopping: { readonly generation: number }; + /** `crash` marks a generation whose runtime exited on its own: its cleanup ends in `Failed`. */ + Stopping: { readonly generation: number; readonly crash?: { readonly cause: unknown } }; Failed: { readonly generation: number; readonly cause: unknown }; }>; const Phase = Data.taggedEnum(); @@ -206,6 +207,11 @@ export type LifecycleEvent = Data.TaggedEnum<{ SessionAvailable: { readonly id: string; readonly generation: number }; LaunchSucceeded: { readonly id: string; readonly generation: number }; LaunchFailed: { readonly id: string; readonly generation: number; readonly cause: unknown }; + /** + * A generation's runtime exited on its own and its cleanup has begun. Reported before the + * cleanup runs, so nothing routes to the dead session while its resources are still held. + */ + SessionLost: { readonly id: string; readonly generation: number; readonly cause: unknown }; Exited: { readonly id: string; readonly generation: number; @@ -715,15 +721,42 @@ const applyEvent = ( break; return [recordFailure(state, id, generation, cause, now, commands), commands]; } + case "SessionLost": { + const { id, generation, cause } = event; + const service = state.services.get(id); + if ( + service === undefined || + (service.phase._tag !== "Starting" && service.phase._tag !== "Running") || + service.phase.generation !== generation + ) + break; + // The crash counts and fails its waiters now; the generation then winds down like a stop. + const failed = recordFailure(state, id, generation, cause, now, commands); + return [ + setService(failed, id, (s) => ({ + ...s, + phase: Phase.Stopping({ generation, crash: { cause } }), + })), + commands, + ]; + } case "Exited": { - const { id, generation, cause, requested } = event; + const { id, generation, requested } = event; const service = state.services.get(id); if (service === undefined || generationOf(service.phase) !== generation) break; if (service.phase._tag === "Stopping" || requested) { + // An explicit stop that arrived during a crash's cleanup wins: it ends `Stopped`. + const crash = + service.phase._tag === "Stopping" && service.intent !== "stopped" + ? service.phase.crash + : undefined; return [ setService(state, id, (s) => ({ ...s, - phase: Phase.Stopped(), + phase: + crash === undefined + ? Phase.Stopped() + : Phase.Failed({ generation, cause: crash.cause }), idleArmedEpoch: undefined, readinessFailure: undefined, reprobing: false, @@ -732,8 +765,6 @@ const applyEvent = ( commands, ]; } - if (service.phase._tag === "Running" || service.phase._tag === "Starting") - return [recordFailure(state, id, generation, cause, now, commands), commands]; break; } case "StopFailed": { diff --git a/packages/stack/src/Lifecycle.unit.test.ts b/packages/stack/src/Lifecycle.unit.test.ts index f3f0b9339a..1590a002fb 100644 --- a/packages/stack/src/Lifecycle.unit.test.ts +++ b/packages/stack/src/Lifecycle.unit.test.ts @@ -36,6 +36,12 @@ const run = ( return { state, commandsByStep }; }; +/** A runtime's unexpected exit as the service reports it: lost first, then cleaned up. */ +const crashSteps = (id: string, generation: number, now: number, cause = "crash") => [ + { event: LifecycleEvent.SessionLost({ id, generation, cause }), now }, + { event: LifecycleEvent.Exited({ id, generation, cause, requested: false }), now }, +]; + const tagsOf = (commands: ReadonlyArray | undefined) => (commands ?? []).map((command) => command._tag); @@ -95,11 +101,13 @@ describe("Lifecycle reducer", () => { }), now: 30, }, + { event: LifecycleEvent.SessionLost({ id: "api", generation: 99, cause: "boom" }), now: 40 }, ]); expect(commandsByStep[1]).toEqual([]); expect(commandsByStep[2]).toEqual([]); expect(commandsByStep[3]).toEqual([]); + expect(commandsByStep[4]).toEqual([]); expect(state.services.get("api")?.phase).toEqual({ _tag: "Starting", generation: 1, @@ -446,15 +454,7 @@ describe("Lifecycle reducer", () => { const generation = startingGeneration(state, "flaky"); ({ state } = run(state, [ { event: LifecycleEvent.LaunchSucceeded({ id: "flaky", generation }), now: now + 1 }, - { - event: LifecycleEvent.Exited({ - id: "flaky", - generation, - cause: "crash", - requested: false, - }), - now: now + 2, - }, + ...crashSteps("flaky", generation, now + 2), ])); return state; }; @@ -623,20 +623,131 @@ describe("Lifecycle reducer", () => { const { state: idle } = run(ready, [ { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 15 }, ]); - const { state: afterExit, commandsByStep: exitCommands } = run(idle, [ - { - event: LifecycleEvent.Exited({ - id: "api", - generation: 1, - cause: "crash", - requested: false, - }), - now: 20, - }, - ]); + const { state: afterExit, commandsByStep: exitCommands } = run(idle, crashSteps("api", 1, 20)); expect(afterExit.services.get("api")?.phase).toMatchObject({ _tag: "Failed", cause: "crash" }); expect(afterExit.services.get("api")?.breaker.consecutiveFailures).toBe(1); - expect(exitCommands[0]).toEqual([]); + expect(exitCommands).toEqual([[], []]); + }); + + describe("crash cleanup", () => { + const sessionLost = (id: string, generation: number, now: number) => ({ + event: LifecycleEvent.SessionLost({ id, generation, cause: "crash" }), + now, + }); + + it("stops routing to a crashed session at once and re-wakes it for traffic that arrived during the cleanup", () => { + let state = initialState(makeGraph([lazy("api")])); + ({ state } = run(state, [ + { event: open("api", 1), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 1 }, + { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 2 }, + ])); + + let commandsByStep: ReadonlyArray>; + ({ state, commandsByStep } = run(state, [ + sessionLost("api", 1, 10), + { event: open("api", 2), now: 11 }, + { event: LifecycleEvent.SessionAvailable({ id: "api", generation: 1 }), now: 12 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 13 }, + ])); + expect(commandsByStep).toEqual([[], [], [], []]); + expect(state.services.get("api")?.phase).toMatchObject({ _tag: "Stopping", generation: 1 }); + expect(state.services.get("api")?.waiters.size).toBe(1); + expect(state.services.get("api")?.breaker.consecutiveFailures).toBe(1); + + ({ state, commandsByStep } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "api", + generation: 1, + cause: "crash", + requested: false, + }), + now: 20, + }, + ])); + expect(commandTagged(at(commandsByStep, 0), "Launch").generation).toBe(2); + expect(state.services.get("api")?.breaker.consecutiveFailures).toBe(1); + }); + + it("fails the waiters of a generation that crashes while starting and ignores its late session", () => { + let state = initialState(makeGraph([lazy("api")])); + let commandsByStep: ReadonlyArray>; + ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); + const generation = startingGeneration(state, "api"); + + ({ state, commandsByStep } = run(state, [ + sessionLost("api", generation, 5), + { event: LifecycleEvent.SessionAvailable({ id: "api", generation }), now: 6 }, + { + event: LifecycleEvent.Exited({ id: "api", generation, cause: "crash", requested: false }), + now: 7, + }, + ])); + expect(commandTagged(at(commandsByStep, 0), "FailConnection")).toMatchObject({ + waiterId: 1, + cause: "crash", + }); + expect(commandsByStep[1]).toEqual([]); + expect(commandsByStep[2]).toEqual([]); + expect(state.services.get("api")?.phase).toMatchObject({ _tag: "Failed", cause: "crash" }); + expect(state.services.get("api")?.breaker.consecutiveFailures).toBe(1); + }); + + it("counts a crash once when its cleanup fails and a retried stop then confirms it", () => { + let state = initialState(makeGraph([lazy("api")])); + ({ state } = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "api" }), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 1 }, + sessionLost("api", 1, 2), + { + event: LifecycleEvent.StopFailed({ + id: "api", + generation: 1, + cause: "busy", + failure: "crash", + }), + now: 3, + }, + ])); + expect(state.services.get("api")?.breaker.consecutiveFailures).toBe(1); + + ({ state } = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "api" }), now: 4 }, + { + event: LifecycleEvent.Exited({ + id: "api", + generation: 1, + cause: undefined, + requested: true, + }), + now: 5, + }, + ])); + expect(state.services.get("api")?.phase._tag).toBe("Stopped"); + expect(state.services.get("api")?.breaker.consecutiveFailures).toBe(1); + }); + + it("ends Stopped when an explicit stop arrives during the crash's cleanup", () => { + let state = initialState(makeGraph([lazy("api")])); + ({ state } = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "api" }), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 1 }, + sessionLost("api", 1, 2), + { event: LifecycleEvent.StopRequested({ id: "api" }), now: 3 }, + { + event: LifecycleEvent.Exited({ + id: "api", + generation: 1, + cause: "crash", + requested: false, + }), + now: 4, + }, + ])); + expect(state.services.get("api")?.phase._tag).toBe("Stopped"); + expect(storageAdmitted(state, "api")).toBe(true); + }); }); it("rejects stopping or restarting a prerequisite while a dependent is active", () => { @@ -999,12 +1110,7 @@ describe("Lifecycle reducer", () => { // until the third failure opens the breaker and a finally rests in `Failed`. const crash = (now: number) => { const generation = currentGeneration(state, "a"); - ({ state } = run(state, [ - { - event: LifecycleEvent.Exited({ id: "a", generation, cause: "crash", requested: false }), - now, - }, - ])); + ({ state } = run(state, crashSteps("a", generation, now))); }; crash(30); crash(31); @@ -1347,19 +1453,8 @@ describe("Lifecycle reducer", () => { expect(tagsOf(commandsByStep[1])).toEqual([]); ({ state } = run(state, [{ event: open("studio", 2), now: 5 }])); - ({ commandsByStep } = run(state, [ - { - event: LifecycleEvent.Exited({ - id: "analytics", - generation: 1, - cause: "gone", - requested: false, - }), - now: 6, - }, - lost(7), - ])); - expect(tagsOf(commandsByStep[1]).includes("Reprobe")).toBe(false); + ({ commandsByStep } = run(state, [...crashSteps("analytics", 1, 6, "gone"), lost(7)])); + expect(tagsOf(commandsByStep[2]).includes("Reprobe")).toBe(false); }); }); diff --git a/packages/stack/src/Orchestrator.integration.test.ts b/packages/stack/src/Orchestrator.integration.test.ts index b428597721..43cf1cde67 100644 --- a/packages/stack/src/Orchestrator.integration.test.ts +++ b/packages/stack/src/Orchestrator.integration.test.ts @@ -702,6 +702,54 @@ describe("service composition", () => { ).pipe(Effect.provide(TestClock.layer())), ); + it.live("holds traffic through a crashed workload's cleanup, then re-wakes it", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const crashed = yield* Deferred.make>(); + const cleanupStarted = yield* Deferred.make(); + const cleanupGate = yield* Deferred.make(); + const rest = yield* makeInstance(orchestrator, "rest", { + exit: crashed, + stop: Deferred.succeed(cleanupStarted, undefined).pipe( + Effect.andThen(Deferred.await(cleanupGate)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "rest", activation: "lazy", idleMillis: 10_000 }], + dependencies: [], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("rest")); + const firstLaunch = (yield* rest.status).launchId; + + yield* Deferred.succeed( + crashed, + Exit.fail(new ServiceError({ operation: "process", message: "crashed" })), + ); + yield* Deferred.await(cleanupStarted); + expect((yield* rest.status).lifecycle).toBe("stopping"); + + const admitted = yield* Deferred.make(); + const waking = yield* Effect.scoped(orchestrator.acquire("rest")).pipe( + Effect.andThen(Deferred.succeed(admitted, undefined)), + Effect.forkChild({ startImmediately: true }), + ); + yield* TestClock.adjust("1 second"); + expect(yield* Deferred.isDone(admitted)).toBe(false); + expect(yield* Ref.get(rest.starts)).toHaveLength(1); + + yield* Deferred.succeed(cleanupGate, undefined); + yield* Fiber.join(waking); + expect(yield* Ref.get(rest.starts)).toHaveLength(2); + const status = yield* rest.status; + expect(status.lifecycle).toBe("running"); + expect(status.launchId).not.toBe(firstLaunch); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + it.live("continues namespace destruction after a service data removal fails", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index a3463240f4..2db71e3fbe 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -235,7 +235,10 @@ export const makeService = ( ); }); - /** Cleans a session whose runtime exited on its own, then reports the exit. */ + /** + * Reports a session whose runtime exited on its own as lost before anything else, so traffic + * stops routing to it while its cleanup runs; then cleans it and reports the exit. + */ const observeExit = Effect.fn("Service.observeExit")(function* ( record: SessionRecord, exit: Exit.Exit, @@ -244,6 +247,10 @@ export const makeService = ( const error = exitError("exit", exit) ?? new ServiceError({ operation: "exit", message: "Runtime exited unexpectedly" }); + if ((yield* Ref.get(current)) !== record) return; + yield* report( + LifecycleEvent.SessionLost({ id, generation: record.generation, cause: error }), + ); const cleaned = yield* execution.withPermit( Effect.gen(function* () { if ((yield* Ref.get(current)) !== record) return undefined; diff --git a/packages/stack/tests/whole-stack/idle.ts b/packages/stack/tests/whole-stack/idle.ts index cc7e56b68b..221fcf9c6e 100644 --- a/packages/stack/tests/whole-stack/idle.ts +++ b/packages/stack/tests/whole-stack/idle.ts @@ -1,5 +1,5 @@ import { expect } from "@effect/vitest"; -import { Context, Effect, Fiber, Layer, Redacted, Schema } from "effect"; +import { Context, Effect, Exit, Fiber, Layer, Option, Redacted, Schema, Stream } from "effect"; import { PgClient } from "@effect/sql-pg"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { SignJWT } from "jose"; @@ -14,6 +14,7 @@ import { type WholeStack, } from "./fixture.ts"; import { subscribeRealtime } from "./websocket.ts"; +import { killWorkload } from "./workloads.ts"; class IdleProbeError extends Schema.TaggedError()("IdleProbeError", { message: Schema.String, @@ -234,7 +235,23 @@ export const restartIdleWake = (runtime: Runtime) => const rewarm = yield* requestWithHeaders(restUrl, headers); expect(rewarm.status, rewarm.body).toBe(200); - expect((yield* rest.status).lifecycle).toBe("running"); + const awake = yield* rest.status; + expect(awake.lifecycle).toBe("running"); + + yield* killWorkload(runtime, fixture, rest.id); + const crashed = yield* rest.followStatus.pipe( + Stream.filter((status) => status.lifecycle === "stopped"), + Stream.runHead, + Effect.map(Option.getOrThrow), + ); + expect(crashed.exit !== undefined && Exit.isFailure(crashed.exit)).toBe(true); + expect(crashed.wakeEnabled).toBe(true); + + const rewake = yield* requestWithHeaders(restUrl, headers); + expect(rewake.status, rewake.body).toBe(200); + const woken = yield* rest.status; + expect(woken.lifecycle).toBe("running"); + expect(woken.launchId).toBeGreaterThan(awake.launchId ?? Number.POSITIVE_INFINITY); }), ); diff --git a/packages/stack/tests/whole-stack/workloads.ts b/packages/stack/tests/whole-stack/workloads.ts index ae29dcf0d4..65c1c442af 100644 --- a/packages/stack/tests/whole-stack/workloads.ts +++ b/packages/stack/tests/whole-stack/workloads.ts @@ -95,6 +95,36 @@ export const captureWorkloads = Effect.fn("WholeStack.captureWorkloads")( snapshotOutput(runtime, fixture, includeStopped), ); +/** + * Kills one instance's workload the way a crash would: SIGKILL of the native workload process + * (the launcher's child, so the launcher survives to report the exit), or `kill` of its container. + */ +export const killWorkload = Effect.fn("WholeStack.killWorkload")( + (runtime: Runtime, fixture: WholeStack, instanceId: string) => + Effect.gen(function* () { + const { identities } = yield* snapshotOutput(runtime, fixture, false); + if (runtime !== "native") { + const container = identities + .map((identity) => identity.split(" ")) + .find(([, instance]) => instance === instanceId)?.[0]; + if (container === undefined) return yield* Effect.die(`No container for ${instanceId}`); + yield* commandOutput(runtime, ["kill", container]); + return; + } + const launcher = identities + .map((identity) => /^(\d+)\s.*supabase-workload-id=(\S+)$/.exec(identity)) + .find((match) => match?.[2] === instanceId)?.[1]; + if (launcher === undefined) return yield* Effect.die(`No launcher for ${instanceId}`); + const children = (yield* commandOutput("ps", ["-axo", "pid=,ppid="])) + .split("\n") + .map((line) => /^\s*(\d+)\s+(\d+)\s*$/.exec(line)) + .filter((match) => match?.[2] === launcher) + .map((match) => match?.[1] ?? ""); + if (children.length === 0) return yield* Effect.die(`No workload under launcher ${launcher}`); + yield* commandOutput("kill", ["-9", ...children]); + }), +); + export const assertWorkloadsGone = Effect.fn("WholeStack.assertWorkloadsGone")( (runtime: Runtime, fixture: WholeStack, snapshot: WorkloadSnapshot, includeStopped = false) => Effect.gen(function* () { From 824c492ec7006505327e1e8d05ef6040bbef143d Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 19:06:21 +0200 Subject: [PATCH 110/142] refactor(stack): let the lifecycle reducer own restart candidates and graph order Restart candidates ride on the reducer's Launch command instead of a side map fenced by generation, composition order comes from the lifecycle graph, and an individual destroy unregisters once removal is confirmed even if the final close fails. --- packages/stack/src/Lifecycle.ts | 22 ++- packages/stack/src/Lifecycle.unit.test.ts | 46 +++-- .../src/Orchestrator.integration.test.ts | 38 +++- packages/stack/src/Orchestrator.ts | 162 ++++++------------ packages/stack/src/Owner.ts | 7 +- packages/stack/src/Service.ts | 11 +- 6 files changed, 136 insertions(+), 150 deletions(-) diff --git a/packages/stack/src/Lifecycle.ts b/packages/stack/src/Lifecycle.ts index 5ba9ece430..575653449f 100644 --- a/packages/stack/src/Lifecycle.ts +++ b/packages/stack/src/Lifecycle.ts @@ -150,6 +150,8 @@ export interface ServiceState { * held and no stop is in flight. A stop, restart, start or new waiter retries the cleanup. */ readonly cleanupFailure: { readonly cause: unknown } | undefined; + /** The newest restart candidate, consumed by the next `Launch` command. */ + readonly restartCandidate: unknown; } const initialService = (spec: ServiceSpec): ServiceState => ({ @@ -166,6 +168,7 @@ const initialService = (spec: ServiceSpec): ServiceState => ({ readinessFailure: undefined, reprobing: false, cleanupFailure: undefined, + restartCandidate: undefined, }); export interface LifecycleState { @@ -212,12 +215,7 @@ export type LifecycleEvent = Data.TaggedEnum<{ * cleanup runs, so nothing routes to the dead session while its resources are still held. */ SessionLost: { readonly id: string; readonly generation: number; readonly cause: unknown }; - Exited: { - readonly id: string; - readonly generation: number; - readonly cause: unknown; - readonly requested: boolean; - }; + Exited: { readonly id: string; readonly generation: number; readonly requested: boolean }; /** * A stop or a failed launch's cleanup of `generation` failed; its resources are still held. * `failure` carries the launch failure or crash that ended the generation, if one did, so it @@ -239,7 +237,8 @@ export type LifecycleEvent = Data.TaggedEnum<{ /** Arms a service for demand-driven launches without forcing one. */ ArmRequested: { readonly id: string }; StopRequested: { readonly id: string }; - RestartRequested: { readonly id: string }; + /** `candidate`, when given, is the launch input the next launch uses in place of the saved one. */ + RestartRequested: { readonly id: string; readonly candidate?: unknown }; StorageReserved: { readonly id: string }; StorageReleased: { readonly id: string }; /** Stops the service and reserves its storage for data removal once both are free. */ @@ -256,7 +255,8 @@ export type LifecycleEvent = Data.TaggedEnum<{ export const LifecycleEvent = Data.taggedEnum(); export type LifecycleCommand = Data.TaggedEnum<{ - Launch: { readonly id: string; readonly generation: number }; + /** `candidate` is the restart's launch input, or undefined when the saved one applies. */ + Launch: { readonly id: string; readonly generation: number; readonly candidate: unknown }; Stop: { readonly id: string; readonly generation: number }; ArmIdleTimer: { readonly id: string; @@ -987,6 +987,7 @@ const applyEvent = ( intent: spec.activation, breaker: initialBreaker, relaunchForced: true, + restartCandidate: event.candidate === undefined ? s.restartCandidate : event.candidate, phase: winding && generation !== undefined ? Phase.Stopping({ generation }) : s.phase, })); return [rearmPrerequisites(retryCleanup(updated, id, commands), id), commands]; @@ -1123,12 +1124,15 @@ const settle = ( if (!prerequisitesSatisfied(next, id)) continue; if (!hasDemand(next, id) && !service.relaunchForced) continue; const generation = next.generationCounters.get(id) ?? 1; - emitted.push(LifecycleCommand.Launch({ id, generation })); + emitted.push( + LifecycleCommand.Launch({ id, generation, candidate: service.restartCandidate }), + ); next = { ...setService(next, id, (s) => ({ ...s, phase: Phase.Starting({ generation, stage: "preparing" }), relaunchForced: false, + restartCandidate: undefined, readinessFailure: undefined, reprobing: false, })), diff --git a/packages/stack/src/Lifecycle.unit.test.ts b/packages/stack/src/Lifecycle.unit.test.ts index 1590a002fb..de6c97be45 100644 --- a/packages/stack/src/Lifecycle.unit.test.ts +++ b/packages/stack/src/Lifecycle.unit.test.ts @@ -39,7 +39,7 @@ const run = ( /** A runtime's unexpected exit as the service reports it: lost first, then cleaned up. */ const crashSteps = (id: string, generation: number, now: number, cause = "crash") => [ { event: LifecycleEvent.SessionLost({ id, generation, cause }), now }, - { event: LifecycleEvent.Exited({ id, generation, cause, requested: false }), now }, + { event: LifecycleEvent.Exited({ id, generation, requested: false }), now }, ]; const tagsOf = (commands: ReadonlyArray | undefined) => @@ -96,7 +96,6 @@ describe("Lifecycle reducer", () => { event: LifecycleEvent.Exited({ id: "api", generation: 99, - cause: "boom", requested: false, }), now: 30, @@ -168,7 +167,6 @@ describe("Lifecycle reducer", () => { event: LifecycleEvent.Exited({ id: "api", generation: 1, - cause: undefined, requested: true, }), now: 1_040, @@ -242,7 +240,6 @@ describe("Lifecycle reducer", () => { event: LifecycleEvent.Exited({ id: "studio", generation: 1, - cause: undefined, requested: true, }), now: 1_060, @@ -269,7 +266,6 @@ describe("Lifecycle reducer", () => { event: LifecycleEvent.Exited({ id: "pg-meta", generation: 1, - cause: undefined, requested: true, }), now: 2_070, @@ -283,7 +279,6 @@ describe("Lifecycle reducer", () => { event: LifecycleEvent.Exited({ id: "functions", generation: 1, - cause: undefined, requested: true, }), now: 2_080, @@ -305,7 +300,6 @@ describe("Lifecycle reducer", () => { event: LifecycleEvent.Exited({ id: "database", generation: 1, - cause: undefined, requested: true, }), now: 3_090, @@ -381,6 +375,34 @@ describe("Lifecycle reducer", () => { expect(tagsOf(commandsByStep[0])).toEqual(["Launch"]); expect(commandTagged(at(commandsByStep, 0), "Launch").id).toBe("a"); }); + + it("launches the newest restart candidate once, then returns to the saved input", () => { + const graph = makeGraph([lazy("api")]); + const { commandsByStep } = run(initialState(graph), [ + { event: open("api", 1), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 1 }, + { event: LifecycleEvent.RestartRequested({ id: "api", candidate: "first" }), now: 2 }, + { event: LifecycleEvent.RestartRequested({ id: "api", candidate: "second" }), now: 3 }, + { event: LifecycleEvent.RestartRequested({ id: "api" }), now: 4 }, + { event: LifecycleEvent.Exited({ id: "api", generation: 1, requested: true }), now: 5 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 2 }), now: 6 }, + { event: LifecycleEvent.StopRequested({ id: "api" }), now: 7 }, + { event: LifecycleEvent.Exited({ id: "api", generation: 2, requested: true }), now: 8 }, + { event: LifecycleEvent.StartRequested({ id: "api" }), now: 9 }, + ]); + expect(commandTagged(at(commandsByStep, 0), "Launch").candidate).toBeUndefined(); + expect(commandTagged(at(commandsByStep, 5), "Launch").candidate).toBe("second"); + expect(commandTagged(at(commandsByStep, 9), "Launch").candidate).toBeUndefined(); + }); + + it("keeps no candidate for a restart the reducer rejects", () => { + const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); + const { state, commandsByStep } = run(initialState(graph), [ + { event: LifecycleEvent.RestartRequested({ id: "a", candidate: "rejected" }), now: 0 }, + ]); + expect(tagsOf(commandsByStep[0])).toEqual(["RequestRejected"]); + expect(state.services.get("a")?.restartCandidate).toBeUndefined(); + }); }); describe("circuit breaker", () => { @@ -660,7 +682,6 @@ describe("Lifecycle reducer", () => { event: LifecycleEvent.Exited({ id: "api", generation: 1, - cause: "crash", requested: false, }), now: 20, @@ -680,7 +701,7 @@ describe("Lifecycle reducer", () => { sessionLost("api", generation, 5), { event: LifecycleEvent.SessionAvailable({ id: "api", generation }), now: 6 }, { - event: LifecycleEvent.Exited({ id: "api", generation, cause: "crash", requested: false }), + event: LifecycleEvent.Exited({ id: "api", generation, requested: false }), now: 7, }, ])); @@ -718,7 +739,6 @@ describe("Lifecycle reducer", () => { event: LifecycleEvent.Exited({ id: "api", generation: 1, - cause: undefined, requested: true, }), now: 5, @@ -739,7 +759,6 @@ describe("Lifecycle reducer", () => { event: LifecycleEvent.Exited({ id: "api", generation: 1, - cause: "crash", requested: false, }), now: 4, @@ -878,7 +897,6 @@ describe("Lifecycle reducer", () => { event: LifecycleEvent.Exited({ id: "database", generation, - cause: undefined, requested: true, }), now: 2, @@ -963,7 +981,6 @@ describe("Lifecycle reducer", () => { event: LifecycleEvent.Exited({ id: "api", generation: 1, - cause: undefined, requested: true, }), now: 31, @@ -1151,7 +1168,6 @@ describe("Lifecycle reducer", () => { event: LifecycleEvent.Exited({ id: "b", generation: 1, - cause: undefined, requested: true, }), now: 1_040, @@ -1214,7 +1230,6 @@ describe("Lifecycle reducer", () => { event: LifecycleEvent.Exited({ id: "api", generation: 1, - cause: undefined, requested: true, }), now: 30, @@ -1497,7 +1512,6 @@ describe("Lifecycle reducer", () => { event: LifecycleEvent.Exited({ id: "api", generation: 1, - cause: undefined, requested: true, }), now: 7, diff --git a/packages/stack/src/Orchestrator.integration.test.ts b/packages/stack/src/Orchestrator.integration.test.ts index 43cf1cde67..3ad7d87b89 100644 --- a/packages/stack/src/Orchestrator.integration.test.ts +++ b/packages/stack/src/Orchestrator.integration.test.ts @@ -42,6 +42,8 @@ const makeInstance = ( readonly launch?: Effect.Effect; readonly stop?: Effect.Effect; readonly removeData?: Effect.Effect; + /** Replaces the default close, which only tracks the bound state. */ + readonly close?: Effect.Effect; /** Resolves the first launch's runtime exit; later launches exit only when stopped. */ readonly exit?: Deferred.Deferred>; /** Every launch's runtime exit, for workloads that crash on demand. */ @@ -106,12 +108,16 @@ const makeInstance = ( confirmRemoved: Effect.void, release: Effect.void, releasePorts: Effect.void, - close: orchestrator.status(id).pipe( - Effect.flatMap((state) => - state.lifecycle === "stopped" && !state.wakeEnabled ? Ref.set(bound, false) : Effect.void, + close: + options.close ?? + orchestrator.status(id).pipe( + Effect.flatMap((state) => + state.lifecycle === "stopped" && !state.wakeEnabled + ? Ref.set(bound, false) + : Effect.void, + ), + Effect.mapError((error) => failure(error.message)), ), - Effect.mapError((error) => failure(error.message)), - ), hasEndpoint: options.endpoint ?? true, inputs: ["databaseUrl"], outputs: { url: Effect.succeed(`postgres://${id}`) }, @@ -822,6 +828,28 @@ describe("destroying an instance with admitted traffic", () => { }), ), ); + + it.live( + "unregisters an instance whose data removal was confirmed even if its final close fails", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const removed = yield* Ref.make(false); + yield* makeInstance(orchestrator, "database", { + removeData: Ref.set(removed, true), + close: Ref.get(removed).pipe( + Effect.flatMap((gone) => (gone ? Effect.fail(failure("close failed")) : Effect.void)), + ), + }); + + const error = yield* orchestrator.destroy("database").pipe(Effect.flip); + + expect(error.message).toBe("close failed"); + expect(yield* orchestrator.instances).toEqual([]); + }), + ), + ); }); describe("wake and idle sleep", () => { diff --git a/packages/stack/src/Orchestrator.ts b/packages/stack/src/Orchestrator.ts index 56d1f9f2fb..0f3d15f8da 100644 --- a/packages/stack/src/Orchestrator.ts +++ b/packages/stack/src/Orchestrator.ts @@ -204,11 +204,6 @@ export interface Interface Effect.Effect; } -interface Graph { - readonly members: ReadonlyMap; - readonly prerequisites: ReadonlyMap>; -} - interface Waiter { readonly deferred: Deferred.Deferred; } @@ -222,30 +217,22 @@ interface Applied { const graphError = (operation: OrchestratorOperation, message: string, cause?: unknown) => new OrchestratorError({ operation, message, ...(cause === undefined ? {} : { cause }) }); -const topo = ( - ids: ReadonlyArray, - prerequisites: ReadonlyMap>, -): ReadonlyArray => { - const result: Array = []; - const seen = new Set(); - const visit = (id: string): void => { - if (seen.has(id)) return; - seen.add(id); - for (const prerequisite of prerequisites.get(id) ?? []) visit(prerequisite); - result.push(id); - }; - for (const id of ids) visit(id); - return result; -}; +/** The composition without `id`, as a member or as either end of a dependency. */ +export const withoutMember = (config: CompositionConfig, id: string): CompositionConfig => ({ + members: config.members.filter((member) => member.id !== id), + dependencies: config.dependencies.filter( + (dependency) => dependency.from !== id && dependency.to !== id, + ), +}); -const validateGraph = ( +const validateComposition = ( configuration: CompositionConfig, values: ReadonlyMap, -): Graph | OrchestratorError => { - const members = new Map(); +): OrchestratorError | undefined => { + const memberIds = new Set(); for (const member of configuration.members) { - if (members.has(member.id)) return graphError("configure", `Duplicate member ${member.id}`); - members.set(member.id, member); + if (memberIds.has(member.id)) return graphError("configure", `Duplicate member ${member.id}`); + memberIds.add(member.id); } const boundInputs = new Set(); const prerequisites = new Map>(); @@ -296,7 +283,6 @@ const validateGraph = ( const failure = visit(id); if (failure !== undefined) return failure; } - return { members, prerequisites }; }; /** Every registered instance as the reducer sees it; a non-member runs as an eager standalone. */ @@ -316,6 +302,15 @@ const lifecycleGraph = (ids: Iterable, configuration: CompositionConfig) }), ); +/** The listed instances, prerequisites first. */ +const inOrder = (graph: LifecycleGraph, ids: Iterable): ReadonlyArray => { + const wanted = new Set(ids); + return graph.order.filter((id) => wanted.has(id)); +}; + +const prerequisitesOf = (graph: LifecycleGraph, id: string): ReadonlyArray => + inOrder(graph, graph.prerequisiteClosure.get(id) ?? []); + const lifecycleOf = (state: LifecycleState, id: string): Status["lifecycle"] => { const phase = state.services.get(id)?.phase; switch (phase?._tag) { @@ -412,9 +407,6 @@ export const make = Effect.fn("Orchestrator.make")(function* ({ members: [], dependencies: [] }); const waiters = yield* Ref.make>(new Map()); const nextWaiterId = yield* Ref.make(1); - const candidates = yield* Ref.make< - ReadonlyMap - >(new Map()); const idleTimers = yield* FiberMap.make(); // Nothing reduces events once the owner closes, so its pending waiters are released here. yield* Scope.addFinalizer( @@ -522,15 +514,12 @@ export const make = Effect.fn("Orchestrator.make")(function* { - const pending = current.get(id); - if (pending === undefined || generation < pending.minGeneration) return [undefined, current]; - const next = new Map(current); - next.delete(id); - return [pending.value, next]; - }); const started = yield* Effect.gen(function* () { const entry = yield* node(id); const inputs = yield* resolveInputs(id); @@ -575,7 +564,7 @@ export const make = Effect.fn("Orchestrator.make")(function* => - topo( - [...(state.graph.prerequisiteClosure.get(id) ?? [])], - new Map([...state.graph.services.values()].map((spec) => [spec.id, spec.prerequisites])), - ); - /** The readiness waits that bound an explicit operation: every prerequisite, then the target. */ const explicitWaits = Effect.fnUntraced(function* (id: string, targetReady: boolean) { const state = yield* SubscriptionRef.get(lifecycle); return [ - ...prerequisitesOf(state, id).map((prerequisite) => ({ + ...prerequisitesOf(state.graph, id).map((prerequisite) => ({ id: prerequisite, requireReady: true, })), @@ -795,7 +778,8 @@ export const make = Effect.fn("Orchestrator.make")(function* - new Map(current).set(id, { minGeneration, value: config }), - ); - } - // Once admitted, the candidate belongs to the next launch even if this caller's wait fails. - const admitted = yield* Ref.make(false); yield* awaitAll({ - requests: [LifecycleEvent.RestartRequested({ id })], + requests: [LifecycleEvent.RestartRequested({ id, candidate: config })], waits: yield* explicitWaits(id, false), traffic: false, operation: "restart", - onApplied: () => Ref.set(admitted, true), - }).pipe( - Effect.tapError(() => - Ref.get(admitted).pipe( - Effect.flatMap((wasAdmitted) => - wasAdmitted - ? Effect.void - : Ref.update(candidates, (current) => { - const next = new Map(current); - next.delete(id); - return next; - }), - ), - ), - ), - ); + }); }); const storage = (id: string, operation: Effect.Effect) => @@ -946,13 +906,7 @@ export const make = Effect.fn("Orchestrator.make")(function* member.id !== id), - dependencies: configured.dependencies.filter( - (dependency) => dependency.from !== id && dependency.to !== id, - ), - }; + const next = withoutMember(yield* Ref.get(composition), id); // The graph update goes first: it removes the service while the destroy still holds it. const applied = yield* applyLocked([ LifecycleEvent.GraphUpdated({ graph: lifecycleGraph(values.keys(), next) }), @@ -983,7 +937,9 @@ export const make = Effect.fn("Orchestrator.make")(function* restore( Effect.gen(function* () { @@ -1004,13 +960,16 @@ export const make = Effect.fn("Orchestrator.make")(function* - Exit.isSuccess(exit) - ? unregister(id) - : dispatch([LifecycleEvent.DestroyReleased({ id })]), + Effect.onExit(() => + Ref.get(removed).pipe( + Effect.flatMap((confirmed) => + confirmed ? unregister(id) : dispatch([LifecycleEvent.DestroyReleased({ id })]), + ), + ), ), ), ); @@ -1043,14 +1002,13 @@ export const make = Effect.fn("Orchestrator.make")(function* member.id), - structure.prerequisites, - ); - const eager = new Set(order.filter((id) => structure.members.get(id)?.activation !== "lazy")); + const { graph } = yield* SubscriptionRef.get(lifecycle); + const memberIds = (yield* Ref.get(composition)).members.map((member) => member.id); + const order = inOrder(graph, [ + ...memberIds, + ...memberIds.flatMap((id) => prerequisitesOf(graph, id)), + ]); + const eager = new Set(order.filter((id) => graph.services.get(id)?.activation !== "lazy")); const completions = new Map>(); for (const id of order) completions.set(id, yield* Deferred.make()); const bound = new Map>(); @@ -1068,7 +1026,7 @@ export const make = Effect.fn("Orchestrator.make")(function* Effect.gen(function* () { const failedPrerequisites: Array = []; - for (const prerequisite of structure.prerequisites.get(id) ?? []) { + for (const prerequisite of graph.services.get(id)?.prerequisites ?? []) { const completion = completions.get(prerequisite); if (completion !== undefined && !(yield* Deferred.await(completion))) failedPrerequisites.push(prerequisite); @@ -1110,13 +1068,8 @@ export const make = Effect.fn("Orchestrator.make")(function* ) { - const configured = yield* Ref.get(composition); - const structure = validateGraph(configured, yield* Ref.get(registry)); - if (structure instanceof OrchestratorError) return yield* structure; - const order = topo(ids, structure.prerequisites) - .filter((id) => ids.includes(id)) - .toReversed(); - return yield* settleOutcomes("stop", order, stop); + const { graph } = yield* SubscriptionRef.get(lifecycle); + return yield* settleOutcomes("stop", inOrder(graph, ids).toReversed(), stop); }); const stopComposition = Effect.fn("Orchestrator.stopComposition")(function* () { @@ -1133,10 +1086,7 @@ export const make = Effect.fn("Orchestrator.make")(function* destroyInstance(id, "retain").pipe( Effect.exit, @@ -1237,8 +1187,8 @@ export const make = Effect.fn("Orchestrator.make")(function* member.id), ...configuration.members.map((member) => member.id), diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index b83ca01e98..ab2a076d8e 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -245,12 +245,7 @@ const withoutInstance = (current: SavedStack, id: string): SavedStack => withoutUnusedCredentials({ ...current, instances: current.instances.filter((instance) => instance.id !== id), - composition: { - members: current.composition.members.filter((member) => member.id !== id), - dependencies: current.composition.dependencies.filter( - (dependency) => dependency.from !== id && dependency.to !== id, - ), - }, + composition: Orchestrator.withoutMember(current.composition, id), }); const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index 2db71e3fbe..f9887a4773 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -265,7 +265,7 @@ export const makeService = ( const generation = record.generation; yield* report( Exit.isSuccess(cleaned) - ? LifecycleEvent.Exited({ id, generation, cause: error, requested: false }) + ? LifecycleEvent.Exited({ id, generation, requested: false }) : LifecycleEvent.StopFailed({ id, generation, @@ -452,7 +452,7 @@ export const makeService = ( .pipe(Effect.exit); yield* report( Exit.isSuccess(stopped) - ? LifecycleEvent.Exited({ id, generation, cause: undefined, requested: true }) + ? LifecycleEvent.Exited({ id, generation, requested: true }) : LifecycleEvent.StopFailed({ id, generation, cause: Cause.squash(stopped.cause) }), ); }); @@ -498,12 +498,7 @@ export const makeService = ( // A previous failed stop can leave the reducer in `Stopping` forever; report this // retry's termination too, or `NetworkNamespace.release`'s endpoint check never learns. yield* report( - LifecycleEvent.Exited({ - id, - generation: leftover.generation, - cause: undefined, - requested: true, - }), + LifecycleEvent.Exited({ id, generation: leftover.generation, requested: true }), ); } const dataScope = yield* Scope.fork(owner, "parallel"); From c84cf751225101cdef0b023f74a5dd3ebbbd01c9 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 19:09:29 +0200 Subject: [PATCH 111/142] refactor(stack): drop the registered observation now that destroy unregisters on removal --- .../db-config.integration.test.ts | 1 - .../commands/db/dump/dump.integration.test.ts | 1 - .../commands/db/reset/reset.integration.test.ts | 1 - .../generate/generate.integration.test.ts | 1 - .../declarative/sync/sync.integration.test.ts | 1 - .../commands/db/start/start.integration.test.ts | 1 - .../stack/logs/logs.integration.test.ts | 1 - .../stack/prepare/prepare.integration.test.ts | 1 - .../start-export-pointer.integration.test.ts | 1 - .../stack/start/start.integration.test.ts | 1 - .../stack/status/status.integration.test.ts | 1 - .../serve/serve.stack.integration.test.ts | 1 - apps/cli/tests/helpers/storage.ts | 1 - .../stack/src/Orchestrator.integration.test.ts | 9 ++++----- packages/stack/src/Owner.ts | 2 +- packages/stack/src/Rpc.ts | 1 - packages/stack/src/Service.integration.test.ts | 14 +++++++------- packages/stack/src/Service.ts | 16 +++------------- packages/stack/tests/standalone-service.ts | 4 ++-- 19 files changed, 17 insertions(+), 42 deletions(-) diff --git a/apps/cli/src/command-internal/db-config.integration.test.ts b/apps/cli/src/command-internal/db-config.integration.test.ts index 436fe889ef..1ce21541bf 100644 --- a/apps/cli/src/command-internal/db-config.integration.test.ts +++ b/apps/cli/src/command-internal/db-config.integration.test.ts @@ -461,7 +461,6 @@ describe("dbConfigResolver (db-url under the stack backend)", () => { currentOperation: undefined, launchId: undefined, wakeEnabled: state === "running", - registered: true, }), followStatus: Stream.empty, logs: Stream.empty, diff --git a/apps/cli/src/commands/db/dump/dump.integration.test.ts b/apps/cli/src/commands/db/dump/dump.integration.test.ts index 8aeff97c8e..d7f0a8c4f7 100644 --- a/apps/cli/src/commands/db/dump/dump.integration.test.ts +++ b/apps/cli/src/commands/db/dump/dump.integration.test.ts @@ -120,7 +120,6 @@ const managedDumpStackApi = (runtime: "native" | "docker") => { currentOperation: undefined, launchId: undefined, wakeEnabled: true, - registered: true, }), followStatus: Stream.empty, logs: Stream.empty, diff --git a/apps/cli/src/commands/db/reset/reset.integration.test.ts b/apps/cli/src/commands/db/reset/reset.integration.test.ts index d4695f30f2..4df92b184a 100644 --- a/apps/cli/src/commands/db/reset/reset.integration.test.ts +++ b/apps/cli/src/commands/db/reset/reset.integration.test.ts @@ -619,7 +619,6 @@ function makeStackObservation( currentOperation: undefined, launchId: undefined, wakeEnabled: opts.wakeEnabled ?? false, - registered: true, }; } diff --git a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts index 27a8784256..26f53e380a 100644 --- a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts @@ -128,7 +128,6 @@ function generateStackApi(workdir: string) { endpoints: [{ name: "sql", protocol: "tcp", host: "127.0.0.1", port: STACK_GENERATE_PORT }], lifecycle: "running", health: "healthy", - registered: true, wakeEnabled: true, currentOperation: undefined, launchId: undefined, diff --git a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts index df21a46df3..aaa03ce522 100644 --- a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts @@ -130,7 +130,6 @@ function syncStackApi(workdir: string, port: number) { endpoints: [{ name: "sql", protocol: "tcp", host: "127.0.0.1", port }], lifecycle: "running", health: "healthy", - registered: true, wakeEnabled: true, currentOperation: undefined, launchId: undefined, diff --git a/apps/cli/src/commands/db/start/start.integration.test.ts b/apps/cli/src/commands/db/start/start.integration.test.ts index 46c9c19d70..ccb0615fa2 100644 --- a/apps/cli/src/commands/db/start/start.integration.test.ts +++ b/apps/cli/src/commands/db/start/start.integration.test.ts @@ -1629,7 +1629,6 @@ describe("db start stack backend", () => { currentOperation: undefined, launchId: undefined, wakeEnabled: state.running, - registered: true, })), followStatus: Stream.empty, logs: Stream.empty, diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts b/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts index 5caba8c782..f913c43f3b 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts @@ -197,7 +197,6 @@ describe("stack logs", () => { yield* Deferred.await(subscribed); yield* Fiber.interrupt(fiber); expect(yield* Ref.get(released)).toBe(true); - expect((yield* service.status).registered).toBe(true); expect((yield* f.api.discover(f.locations))[0]?.host).toBeDefined(); expect(f.telemetry.flushed).toBe(true); }), diff --git a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts index 565e59b568..7b0192e2f7 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts @@ -109,7 +109,6 @@ const makeFixture = (root: string, options: FixtureOptions = {}) => { currentOperation: undefined, launchId: undefined, wakeEnabled: false, - registered: true, }), followStatus: Stream.empty, logs: Stream.empty, diff --git a/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts index c998e74f68..7ae6e0f6b6 100644 --- a/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts @@ -70,7 +70,6 @@ function makeDatabaseStack(sqlPort: number, credentials: StackCredentials): Stac currentOperation: undefined, launchId: undefined, wakeEnabled: false, - registered: true, }; const databaseInstance = { id: "database-id", diff --git a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts index e3adbca6b7..b15f1c642b 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts @@ -112,7 +112,6 @@ const instance = ( currentOperation: undefined, launchId: undefined, wakeEnabled: view().wakeEnabled, - registered: true, }); const base = { id, diff --git a/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts b/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts index 793649418f..3396d83a97 100644 --- a/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts @@ -107,7 +107,6 @@ const makeObservation = ( currentOperation: undefined, launchId: undefined, wakeEnabled: true, - registered: true, ...input, }); diff --git a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts index c7e2e04fe3..f1c68f27dc 100644 --- a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts +++ b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts @@ -94,7 +94,6 @@ const observation = ( exit: undefined, launchId: 1, wakeEnabled: false, - registered: true, ...overrides, }); diff --git a/apps/cli/tests/helpers/storage.ts b/apps/cli/tests/helpers/storage.ts index 04171be38d..654b6eb7dc 100644 --- a/apps/cli/tests/helpers/storage.ts +++ b/apps/cli/tests/helpers/storage.ts @@ -161,7 +161,6 @@ const observation = ( currentOperation: undefined, launchId: undefined, wakeEnabled, - registered: true, }); const instance = ( diff --git a/packages/stack/src/Orchestrator.integration.test.ts b/packages/stack/src/Orchestrator.integration.test.ts index 3ad7d87b89..d36d4dcac6 100644 --- a/packages/stack/src/Orchestrator.integration.test.ts +++ b/packages/stack/src/Orchestrator.integration.test.ts @@ -762,8 +762,8 @@ describe("service composition", () => { const orchestrator = yield* makeTestOrchestrator(); const events = yield* Ref.make>([]); const failRemoval = yield* Ref.make(true); - const next = yield* makeInstance(orchestrator, "next", { events }); - const failing = yield* makeInstance(orchestrator, "failing", { + yield* makeInstance(orchestrator, "next", { events }); + yield* makeInstance(orchestrator, "failing", { events, removeData: Ref.get(failRemoval).pipe( Effect.flatMap((fail) => @@ -790,12 +790,11 @@ describe("service composition", () => { "destroy:failing", "destroy:next", ]); - expect((yield* failing.core.get).registered).toBe(true); - expect((yield* next.core.get).registered).toBe(false); + expect((yield* orchestrator.instances).map(({ id }) => id)).toEqual(["failing"]); yield* Ref.set(failRemoval, false); yield* orchestrator.destroyNamespace; - expect((yield* failing.core.get).registered).toBe(false); + expect(yield* orchestrator.instances).toEqual([]); }), ), ); diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index ab2a076d8e..3ed29f921e 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -463,7 +463,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ).pipe(Effect.provideService(Scope.Scope, ownerScope)); // Listeners stay open while the service runs or demand can still wake it. const enabled = orchestrator.status(id).pipe( - Effect.map((status) => status.registered && !Orchestrator.isStoppedAndWakeDisabled(status)), + Effect.map((status) => !Orchestrator.isStoppedAndWakeDisabled(status)), Effect.orElseSucceed(() => false), ); const configFor = (inputs: Record, candidate: unknown) => diff --git a/packages/stack/src/Rpc.ts b/packages/stack/src/Rpc.ts index ec0738d96e..0c57dcc1d3 100644 --- a/packages/stack/src/Rpc.ts +++ b/packages/stack/src/Rpc.ts @@ -130,7 +130,6 @@ export const Observation = Schema.Struct({ ), launchId: Schema.UndefinedOr(Schema.Int), wakeEnabled: Schema.Boolean, - registered: Schema.Boolean, }); export interface Observation extends Schema.Schema.Type {} diff --git a/packages/stack/src/Service.integration.test.ts b/packages/stack/src/Service.integration.test.ts index d92c6b39fb..3c9ac4bd15 100644 --- a/packages/stack/src/Service.integration.test.ts +++ b/packages/stack/src/Service.integration.test.ts @@ -389,7 +389,7 @@ describe("service execution", () => { yield* open(original.stopGate); yield* open(original.removeGate); yield* service.destroy; - expect((yield* service.execution).registered).toBe(false); + expect(yield* service.listed).toBe(false); expect((yield* Ref.get(original.state)).removed).toBe(true); yield* open(restartPreparation.gate); @@ -670,7 +670,6 @@ describe("service execution", () => { "the retry's confirmed termination must reach the reducer, or it stays Stopping forever", ).toBe(true); expect((yield* Ref.get(plan.state)).removed).toBe(true); - expect((yield* service.get).registered).toBe(false); }), ), ); @@ -693,14 +692,15 @@ describe("service execution", () => { yield* Ref.set(plan.stopFailure, true); expect(Exit.isFailure(yield* service.destroy.pipe(Effect.exit))).toBe(true); - expect(yield* service.get).toMatchObject({ lifecycle: "stopping", registered: true }); + expect(yield* service.get).toMatchObject({ lifecycle: "stopping" }); + expect(yield* service.listed).toBe(true); expect(yield* Ref.get(dataRemoved)).toBe(false); yield* open(plan.stopGate); yield* open(plan.removeGate); yield* service.destroy; expect(yield* Ref.get(dataRemoved)).toBe(true); - expect((yield* service.execution).registered).toBe(false); + expect(yield* service.listed).toBe(false); }), ), ); @@ -729,7 +729,7 @@ describe("service execution", () => { const destroyed = yield* service.destroy.pipe(Effect.forkScoped); yield* awaitDestroyPending(service); - expect((yield* service.execution).registered).toBe(true); + expect(yield* service.listed).toBe(true); yield* open(storageGate); yield* Fiber.join(storage); yield* Deferred.await(removalStarted); @@ -738,7 +738,7 @@ describe("service execution", () => { yield* open(removalGate); yield* Fiber.join(destroyed); - expect((yield* service.execution).registered).toBe(false); + expect(yield* service.listed).toBe(false); }), ), ); @@ -769,7 +769,7 @@ describe("service execution", () => { expect(yield* service.storage(Effect.succeed("stored"))).toBe("stored"); expect(yield* Ref.get(dataRemoved)).toBe(false); - expect((yield* service.execution).registered).toBe(true); + expect(yield* service.listed).toBe(true); const plan = yield* makeRuntimePlan; yield* Queue.offer(plans, plan); yield* open(plan.launchGate); diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index f9887a4773..74225ff3a2 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -26,7 +26,6 @@ export interface ServiceObservation { readonly error: ServiceError | undefined; readonly exit: Exit.Exit | undefined; readonly currentOperation: ServiceOperation | undefined; - readonly registered: boolean; } export class ServiceError extends Data.TaggedError("ServiceError")<{ @@ -101,11 +100,10 @@ export interface ServiceInstance { operation: Effect.Effect, ) => Effect.Effect; /** - * Removes the instance's data and marks it unregistered; the service must already be stopped. + * Removes the instance's data; the service must already be stopped. * `confirm`, when given, runs inside the same execution lock once resources are confirmed - * removed and before the instance is marked unregistered — destroy's registration publication. - * Omitting it (abandonment) reuses the identical confirmed, serialized cleanup without touching - * any registration. + * removed — destroy's registration publication. Omitting it (abandonment) reuses the identical + * confirmed, serialized cleanup without touching any registration. */ readonly removeData: ( confirm?: Effect.Effect, @@ -143,9 +141,6 @@ const exitError = ( }), ); -const destroyedError = (id: string) => - new ServiceError({ operation: "launch", message: `Service ${id} was destroyed` }); - /** Creates one service's execution boundary; launches and cleanups run in the current scope. */ export const makeService = ( definition: ServiceDefinition, @@ -173,7 +168,6 @@ export const makeService = ( error: undefined, exit: undefined, currentOperation: undefined, - registered: true, }); const update = (change: Partial>) => @@ -282,8 +276,6 @@ export const makeService = ( handle: Scope.Closeable, ) { if (generation <= (yield* Ref.get(stoppedThrough))) return { _tag: "Fenced" } as const; - const observation = yield* SubscriptionRef.get(observations); - if (!observation.registered) return { _tag: "Failed", error: destroyedError(id) } as const; // The lifecycle launches only once the previous session's cleanup is confirmed. if ((yield* Ref.get(current)) !== undefined) return { @@ -489,7 +481,6 @@ export const makeService = ( yield* Effect.annotateCurrentSpan({ member_id: id }); yield* execution.withPermit( Effect.gen(function* () { - if (!(yield* SubscriptionRef.get(observations)).registered) return; yield* update({ currentOperation: "destroy" }); yield* Effect.gen(function* () { const leftover = yield* Ref.get(current); @@ -509,7 +500,6 @@ export const makeService = ( Effect.ensuring(Scope.close(dataScope, Exit.void)), ); yield* confirm; - yield* update({ registered: false }); }).pipe(Effect.ensuring(update({ currentOperation: undefined }))); }), ); diff --git a/packages/stack/tests/standalone-service.ts b/packages/stack/tests/standalone-service.ts index fdd0cc428e..935b26ebfe 100644 --- a/packages/stack/tests/standalone-service.ts +++ b/packages/stack/tests/standalone-service.ts @@ -48,8 +48,8 @@ export const makeStandaloneService = Effect.fn("Test.makeStandaloneService")(fun Effect.map(([status, execution]) => ({ ...status, config: execution.config })), ), observation: orchestrator.changes(id), - /** Execution facts that outlive the instance's registration, such as `registered`. */ - execution: core.get, + /** Whether the orchestrator still lists the instance. */ + listed: orchestrator.instances.pipe(Effect.map((entries) => entries.some((e) => e.id === id))), start: orchestrator.start(id), ready: orchestrator.ready(id), stop: orchestrator.stop(id), From 1b0834d0138bb03ea0ba5da8cf4e9c7b9e055502 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 19:21:16 +0200 Subject: [PATCH 112/142] refactor(stack): end an abandoned stack through the ordinary destroy An owner whose registration vanished now runs the regular destroy once, best effort, instead of a parallel cleanup with its own retry loop. Creation writes are skipped when nothing changed, and a credential change also waits for standalone credential consumers to stop. --- packages/stack/ARCHITECTURE.md | 17 +- packages/stack/src/Orchestrator.ts | 7 +- packages/stack/src/Owner.integration.test.ts | 45 ++++ packages/stack/src/Owner.ts | 204 ++++-------------- .../stack/src/Service.integration.test.ts | 7 +- packages/stack/src/Service.ts | 3 +- ...ost.container-shutdown.integration.test.ts | 26 ++- .../stack/src/StackHost.integration.test.ts | 16 +- packages/stack/src/StackHost.ts | 102 +++------ .../src/runtime/Container.integration.test.ts | 88 -------- packages/stack/src/runtime/Container.ts | 23 +- 11 files changed, 161 insertions(+), 377 deletions(-) diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 9ad8df2d6b..9faa279ded 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -497,13 +497,14 @@ is deleted, moved or unmounted). Every owner polls its own `//sta interval (30 seconds by default) and treats only a confirmed ENOENT as gone; any other read error, such as a permission failure or a transient I/O error on an unmounted root, keeps the owner running and logs a warning instead. On a confirmed-gone registration the owner treats its -ownership as ended: it stops every workload (ordered stop, above) and removes what it -created through the same registration-independent instance cleanup destroy uses, retrying -transient cleanup failures and giving up only once an engine is confirmed unreachable (what it -could not remove is logged, not retried forever), and exits. It never reads or writes the -registration again, and it does not release its port reservations: a stale reservation is -reclaimed lazily by whichever stack next needs the port. Deletion, a move and an unmount are treated identically, because a stack whose root is -unreachable cannot operate regardless of which of the three caused it. +ownership as ended: after any stop already running settles, it runs the ordinary destroy once +(ordered stop, instance data removal, label sweep, port release) and exits. The destroy is best +effort and its failure is logged, not retried. Registration and state root need not exist for it +to complete: confirming the absent registration skips the registry lock, so the label sweep still +runs when the whole state root is gone. A transient engine failure during that single cleanup +leaves labelled containers behind; the next startup, stop or destroy of a stack with the same id +and data root sweeps them by label. Deletion, a move and an unmount are treated identically, +because a stack whose root is unreachable cannot operate regardless of which of the three caused it. **Engine targets.** A container stack pins the engine target it first resolves, so every command it runs reaches the same engine. Docker pins `DOCKER_HOST` as `--host`, else the current context. Podman pins `CONTAINER_HOST` as `--url`, else `CONTAINER_CONNECTION` as `--connection`; with neither set, a bare `podman info` decides whether the engine is local or remote. A remote engine (macOS `podman machine`) pins the default system connection by name, and a local one adds no flags. On rootless Podman, containers that run as the caller's uid with writable borrowed bind mounts get `--userns=keep-id` so files stay owned by the caller. Containers reach host listeners through the engine's host alias, `host.docker.internal` or `host.containers.internal`. @@ -524,7 +525,7 @@ During Draining: 5. For destruction, remove proven-owned data and metadata after shutdown. 6. Send the outcome, close the control endpoint and release ownership. -**Ordered stop.** Stop, destroy and abandonment close no listener up front. Public and internal +**Ordered stop.** Stop and destroy close no listener up front. Public and internal dependency traffic share the same listeners, so a dependent's graceful stop (Vector flushing its last batch to Analytics, for example) still reaches its prerequisite through the stack proxy while it is running. Services stop in reverse dependency order, and each listener closes once its own diff --git a/packages/stack/src/Orchestrator.ts b/packages/stack/src/Orchestrator.ts index 0f3d15f8da..721e1fdf20 100644 --- a/packages/stack/src/Orchestrator.ts +++ b/packages/stack/src/Orchestrator.ts @@ -76,14 +76,13 @@ export interface RegisteredInstance { readonly close: Effect.Effect; /** * Publishes this instance's registration removal; run only once `core.removeData`'s resources - * are confirmed removed, as the `confirm` step of that same execution lock. Destroy passes this; - * abandonment never does, because there is no registration left to update. + * are confirmed removed, as the `confirm` step of that same execution lock. A no-op when the + * registration or its state root is already gone. */ readonly confirmRemoved: Effect.Effect; /** * Closes this instance's network endpoints for good once its data is confirmed removed. Never - * deletes saved port assignments; abandonment never calls it, and a stale reservation stays for - * `Ports.ts`'s lazy reclamation once the holder's registration is confirmed gone. + * deletes saved port assignments. */ readonly release: Effect.Effect; /** Deletes this instance's saved port assignments; an individual destroy runs it, a stack-wide destroy leaves them to `Network.releaseStack`. */ diff --git a/packages/stack/src/Owner.integration.test.ts b/packages/stack/src/Owner.integration.test.ts index ce5a8ea389..7f5fcff9df 100644 --- a/packages/stack/src/Owner.integration.test.ts +++ b/packages/stack/src/Owner.integration.test.ts @@ -928,6 +928,51 @@ it.effect("refuses to generate credentials for a stack whose saved instances con ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); +it.live("blocks a credential change while a standalone credential consumer runs", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-owner-credential-admission-", + }); + const stack = initial("owner-credential-admission"); + const state = yield* stateFor(`${root}/state`); + yield* state.save(stack); + const owner = yield* ownerFor({ saved: stack, state, root: `${root}/data`, cacheRoot }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + const standalone = yield* owner.rpc.createService({ + service: "rest", + config: { databaseUrl: "postgresql://authenticator@127.0.0.1:1/postgres" }, + endpoints: {}, + }); + yield* owner.rpc.startService({ id: standalone.id }); + const before = yield* owner.getStackCredentials; + + const refused = yield* owner.rpc + .supabaseComposition({ + services: [{ service: "mail", config: {}, endpoints: {} }], + keys: { + publishableKey: "publishable-rotated", + secretKey: "secret-rotated", + anonKey: "anon-rotated", + serviceRoleKey: "service-role-rotated", + gotrueJwtKeys: "[]", + publicSigningKeys: "[]", + anonKeyIsOverride: true, + serviceRoleKeyIsOverride: true, + }, + }) + .pipe(Effect.flip); + + expect(refused.message).toContain( + `Service ${standalone.id} must be stopped with wake disabled before stack credentials change`, + ); + expect(yield* owner.getStackCredentials).toEqual(before); + yield* owner.rpc.stopService({ id: standalone.id }); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + it.live("rejects a missing required input before starting or stopping the service", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index 3ed29f921e..c81806dbe8 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -1,10 +1,7 @@ import { - Cause, Context, Crypto, - Duration, Effect, - Exit, Fiber, FileSystem, Layer, @@ -63,7 +60,6 @@ import { } from "./services/Catalog.ts"; import type { CatalogError } from "./services/Recipe.ts"; import * as Container from "./runtime/Container.ts"; -import * as Paths from "./namespace/Paths.ts"; import { projectSegmentFor } from "./identity/Identity.ts"; import { stackError, type OwnerRpc } from "./Rpc.ts"; import * as StackNamespace from "./StackNamespace.ts"; @@ -141,16 +137,7 @@ export interface Interface { * containers, native socket directories and saved state. */ readonly destroy: Effect.Effect; - /** - * Ends ownership after a confirmed-gone registration: stops every workload, and - * removes what it created through the same registration-independent cleanup path as destroy, - * without touching the registration or releasing port reservations. A leftover resource past - * the engine-unreachable backstop is logged, not thrown. - */ - readonly abandon: Effect.Effect; }; - /** Direct probe of the pinned engine target; always `true` for a native stack. */ - readonly engineAvailable: Effect.Effect; } export class Service extends Context.Service()("@supabase/stack/Owner") {} @@ -174,57 +161,6 @@ const serviceError = const rpcError = (operation: string) => Effect.mapError((cause: unknown) => stackError(operation, cause)); -/** - * The first backoff delay {@link retryUntilConfirmed} sleeps between attempts, doubling up to 5 - * seconds. Internal only: a test overrides this reference to observe several attempts without - * waiting through a production-length backoff, never through an env var or `Config`. - */ -export const RetryUntilConfirmedInitialDelay = Context.Reference( - "@supabase/stack/RetryUntilConfirmedInitialDelay", - { defaultValue: () => Duration.millis(200) }, -); - -/** - * Retries `operation` with a bounded, doubling backoff until it succeeds or `engineAvailable` - * resolves `false`: engine availability is a direct probe, never a message or cause - * classification, so this is the one place every abandonment cleanup stage decides whether a - * failure is worth retrying. A `false` probe fails with the operation's own last error; an - * interruption or defect is never retried. - */ -export const retryUntilConfirmed = ( - operation: Effect.Effect, - engineAvailable: Effect.Effect, -): Effect.Effect => - Effect.gen(function* () { - let delay = Duration.fromInputUnsafe(yield* RetryUntilConfirmedInitialDelay); - while (true) { - const result = yield* Effect.exit(operation); - if (Exit.isSuccess(result)) return result.value; - const failure = Cause.findErrorOption(result.cause); - if (Option.isNone(failure)) return yield* Effect.failCause(result.cause); - if (!(yield* engineAvailable)) return yield* Effect.fail(failure.value); - yield* Effect.sleep(delay); - delay = Duration.min(Duration.times(delay, 2), Duration.seconds(5)); - } - }); - -/** - * Cleans up one abandoned instance through the same confirmed, serialized execution path destroy - * uses (`core.removeData`, under the service's execution lock, retrying only steps a previous - * attempt didn't finish), but without a `confirm` step: abandonment never touches the registration. - * Retries transient failures; only a confirmed-unreachable engine is logged and left behind, so a - * recoverable failure never blocks the owner's exit. - */ -const cleanupAbandonedInstance = (entry: Entry, engineAvailable: Effect.Effect) => - retryUntilConfirmed(entry.core.removeData(), engineAvailable).pipe( - Effect.catch((error) => - Effect.logWarning( - `Abandoned stack could not confirm cleanup of ${entry.service} ${entry.id}; leaving it behind`, - error, - ), - ), - ); - const mergeInputs = (creation: ServiceCreation, inputs: Record) => Schema.decodeUnknownEffect(ServiceCreation)({ ...creation, @@ -261,20 +197,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { const crypto = Context.get(services, Crypto.Crypto); const path = Context.get(services, Path.Path); const fs = Context.get(services, FileSystem.FileSystem); - const spawner = Context.get(services, ChildProcessSpawner.ChildProcessSpawner); const network = yield* Network.Service; - // A native stack has no engine to lose, so every cleanup failure is retried forever; a docker - // stack probes the pinned target directly, never by classifying a failure's cause. - const engineAvailable: Effect.Effect = - options.engineTarget === undefined - ? Effect.succeed(true) - : Container.probeEngineReachable(spawner, options.engineTarget); - // Shared per-launch environment-file scratch directory (Container.ts), owned by this stack as a - // whole rather than any one instance; registration-independent so abandonment reaches it - // without `Registry.remove`, which never runs then. - const removeContainerEnvRoot = fs - .remove(path.join(options.root, Paths.CONTAINER_ENV_DIRNAME), { recursive: true, force: true }) - .pipe(Effect.mapError(serviceError("cleanup"))); // Everything under the data root belongs to the stack, including what a killed owner left // mid-command (`jobs/`); the directory itself stays for `Registry.pruneDestroyed`. const removeDataRootEntries = fs.exists(options.root).pipe( @@ -360,24 +283,37 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { readSaved.pipe(Effect.flatMap((current) => options.state.save(update(current)))), ); // Idempotent on an already-missing registration (a deleted or already-destroyed state.json), - // unlike `updateState`: destroy must still be able to publish an instance's removal, or confirm - // there is nothing left to publish, even when the registration disappeared from under it. + // unlike `updateState`: destroy must still be able to confirm there is nothing left to publish + // when the registration, or the whole state root with its registry lock, vanished from under it. const removeInstanceRegistration = (id: string) => - options.state.withLock( - options.state - .read(stackId) - .pipe( - Effect.flatMap((current) => - current === undefined ? Effect.void : options.state.save(withoutInstance(current, id)), - ), + options.state + .read(stackId) + .pipe( + Effect.flatMap((registered) => + registered === undefined + ? Effect.void + : options.state.withLock( + options.state + .read(stackId) + .pipe( + Effect.flatMap((current) => + current === undefined + ? Effect.void + : options.state.save(withoutInstance(current, id)), + ), + ), + ), ), - ); + ); - const requireStopped = (configuration: CompositionConfig) => + const requireStopped = (current: SavedStack) => Effect.forEach( new Set([ - ...configuration.members.map(({ id }) => id), - ...configuration.dependencies.flatMap(({ from, to }) => [from, to]), + ...current.composition.members.map(({ id }) => id), + ...current.composition.dependencies.flatMap(({ from, to }) => [from, to]), + ...current.instances + .filter(({ creation }) => consumesCredentials(creation)) + .map(({ id }) => id), ]), (id) => orchestrator.status(id).pipe( @@ -403,7 +339,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { const current = yield* readSaved; const next = yield* nextCredentials(current, overrides, keys); if (next === current.credentials) return next; - if (current.credentials !== undefined) yield* requireStopped(current.composition); + if (current.credentials !== undefined) yield* requireStopped(current); yield* options.state.save({ ...current, credentials: next }); return next; }), @@ -434,13 +370,20 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ...(options.engineTarget === undefined ? {} : { engineTarget: options.engineTarget }), }).pipe(Effect.provideContext(services)); + const sameCreation = Schema.toEquivalence(ServiceCreation); const persistCreation = (entry: Pick, creation: ServiceCreation) => - updateState((current) => ({ - ...current, - instances: current.instances.map((instance) => - instance.id === entry.id ? { ...instance, creation } : instance, + Ref.get(entry.creation).pipe( + Effect.flatMap((saved) => + sameCreation(saved, creation) + ? Effect.void + : updateState((current) => ({ + ...current, + instances: current.instances.map((instance) => + instance.id === entry.id ? { ...instance, creation } : instance, + ), + })).pipe(Effect.andThen(Ref.set(entry.creation, creation))), ), - })).pipe(Effect.andThen(Ref.set(entry.creation, creation))); + ); const register = Effect.fn("Owner.register")(function* (id: string, recipe: CatalogRecipe) { if (Option.isSome(yield* orchestrator.get(id).pipe(Effect.option))) @@ -816,76 +759,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { definitionGate.withPermits(1), Effect.withSpan("Owner.destroyNamespace"), ), - // Settles admitted definition work under the same gate as stop/destroy first. Never - // reads or writes the registration (already confirmed gone), so every step below is - // best-effort and logs rather than fails; the caller must still be able to exit. - abandon: orchestrator.stopNamespace.pipe( - Effect.catch((cause) => - Effect.logWarning("Abandoned stack could not confirm every workload stopped", cause), - ), - Effect.andThen( - orchestrator.instances.pipe( - Effect.flatMap((entries) => - Effect.forEach(entries, (entry) => cleanupAbandonedInstance(entry, engineAvailable), { - concurrency: "unbounded", - discard: true, - }), - ), - ), - ), - // Registration-independent confirming sweep: every container carrying this stack's - // identity and data-root labels, found through the pinned engine directly rather than - // through any in-memory helper registry's own bookkeeping — covers a shared storage - // helper the per-instance cleanup above never removes on its own. The data-root label - // keeps this scoped to this owner's own containers even when another owner shares the - // same stack id under a different root. - Effect.andThen( - options.engineTarget === undefined - ? Effect.void - : retryUntilConfirmed( - Container.removeStackContainers({ - target: options.engineTarget, - stackId, - stackRoot: path.resolve(options.root), - }).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - Effect.mapError(serviceError("cleanup")), - Effect.filterOrFail( - (remaining) => remaining.length === 0, - (remaining) => - new ServiceError({ - operation: "cleanup", - message: `Containers remain: ${remaining.join(", ")}`, - }), - ), - ), - engineAvailable, - ).pipe( - Effect.catch((cause) => - Effect.logWarning( - "Abandoned stack could not confirm every container removed; leaving it behind", - cause, - ), - ), - ), - ), - // Filesystem-only, so always retried to confirmation regardless of engine availability: - // its own `force: true` tolerates an already-gone directory, never a transient failure. - // Port reservations are never released here: a stale reservation stays for `Ports.ts`'s - // own lazy reclamation once the holder's registration is confirmed gone, the abandonment - // trigger, so abandonment never touches `Ports.ts`'s registry at all. - Effect.andThen( - retryUntilConfirmed(removeContainerEnvRoot, Effect.succeed(true)).pipe( - Effect.catch((cause) => - Effect.logWarning("Abandoned stack could not remove its container-env root", cause), - ), - ), - ), - definitionGate.withPermits(1), - Effect.withSpan("Owner.abandonNamespace", { attributes: { stack_id: stackId } }), - ), }, - engineAvailable, } satisfies Interface; }); diff --git a/packages/stack/src/Service.integration.test.ts b/packages/stack/src/Service.integration.test.ts index 3c9ac4bd15..0bd6a45a07 100644 --- a/packages/stack/src/Service.integration.test.ts +++ b/packages/stack/src/Service.integration.test.ts @@ -639,10 +639,9 @@ describe("service execution", () => { () => Effect.scoped( Effect.gen(function* () { - // Direct `makeService`, not `makeStandaloneService`: abandonment calls `core.removeData` - // without ever going through the orchestrator's own dispatch, which already retries and - // reports a failed stop correctly on its own (`Lifecycle.ts`'s `retryCleanup`) and so - // never exercises this path. Only a direct `removeData` call does. + // Direct `makeService`, not `makeStandaloneService`: the orchestrator's own dispatch + // already retries and reports a failed stop correctly (`Lifecycle.ts`'s `retryCleanup`) + // and so never exercises this path. Only a direct `removeData` call does. const plans = yield* Queue.unbounded(); const events = yield* Ref.make>([]); const service = yield* makeService(makeDefinition(plans), { diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index 74225ff3a2..506d338123 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -102,8 +102,7 @@ export interface ServiceInstance { /** * Removes the instance's data; the service must already be stopped. * `confirm`, when given, runs inside the same execution lock once resources are confirmed - * removed — destroy's registration publication. Omitting it (abandonment) reuses the identical - * confirmed, serialized cleanup without touching any registration. + * removed — destroy's registration publication. */ readonly removeData: ( confirm?: Effect.Effect, diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index 5a041aa0f3..028800918a 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -721,6 +721,24 @@ it.live.skipIf(process.platform === "win32")( const status = yield* client.status({ id: mail.id }); const port = status.endpoints.find((endpoint) => endpoint.name === "http")?.port; if (port === undefined) return yield* Effect.die("Missing mail http endpoint"); + // Reachable only by the final label sweep, which runs only if confirming the absent + // registration does not fail on the vanished registry lock. + yield* engine([ + "run", + "-d", + "--name", + `supabase-orphan-${stackId}`, + "--label", + `com.supabase.stack=${stackId}`, + "--label", + `com.supabase.stack-root=${dataRoot}`, + "--label", + "com.supabase.stack-managed=true", + helperImage, + "/bin/sh", + "-c", + "trap : TERM INT; while :; do sleep 3600; done", + ]); // Deletes the actual `` itself, not just the entries inside // `/`: its sibling `.registry-lock.sqlite` (`Ports.ts`'s per-root @@ -739,12 +757,12 @@ it.live.skipIf(process.platform === "win32")( expect( yield* containers(stackId, dataRoot), - "abandonment removes containers from in-memory resources alone", + "abandonment removes the orphan and the service containers with the registry gone", ).toEqual([]); - // The port reservation is never released by abandonment: - // a fresh stack can still claim the exact same port, through `Ports.ts`'s own lazy - // reclamation once the former holder's registration is confirmed gone. + // The vanished registry cannot release the port reservation: a fresh stack can still + // claim the exact same port, through `Ports.ts`'s own lazy reclamation once the former + // holder's registration is confirmed gone. const reclaimedBase = yield* fs.makeTempDirectoryScoped({ prefix: "stack-abandon-docker-root-reclaim-", }); diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts index 35c0a05164..b3d31405dc 100644 --- a/packages/stack/src/StackHost.integration.test.ts +++ b/packages/stack/src/StackHost.integration.test.ts @@ -342,8 +342,8 @@ it.live( cacheRoot: "/tmp/supabase-stack-artifacts", }); const stopGate = yield* Deferred.make(); - const abandonEntered = yield* Deferred.make(); - const abandonGate = yield* Deferred.make(); + const destroyEntered = yield* Deferred.make(); + const destroyGate = yield* Deferred.make(); const failedOwner = { ...owner, namespace: { @@ -353,25 +353,25 @@ it.live( Effect.fail(new OrchestratorError({ operation: "stop", message: "stop failed" })), ), ), - abandon: Deferred.succeed(abandonEntered, undefined).pipe( - Effect.andThen(Deferred.await(abandonGate)), - Effect.andThen(owner.namespace.abandon), + destroy: Deferred.succeed(destroyEntered, undefined).pipe( + Effect.andThen(Deferred.await(destroyGate)), + Effect.andThen(owner.namespace.destroy), ), }, }; const { runtime } = yield* inProcessRuntime(failedOwner, state, root); const stopFiber = yield* Effect.forkScoped(runtime.shutdown(false)); // Abandon is requested while the stop is still in flight, never after it has settled: - // the one shutdown pipeline (pass 3, A) waits for it, instead of racing its own cleanup. + // the one shutdown pipeline waits for it, instead of racing its own cleanup. const abandonFiber = yield* Effect.forkScoped(runtime.abandon); yield* Deferred.succeed(stopGate, undefined); expect(Exit.isFailure(yield* Fiber.join(stopFiber).pipe(Effect.exit))).toBe(true); - yield* Deferred.await(abandonEntered).pipe(Effect.timeout("2 seconds")); + yield* Deferred.await(destroyEntered).pipe(Effect.timeout("2 seconds")); expect( yield* Deferred.isDone(owner.draining), "admission stays closed while abandonment cleans up", ).toBe(true); - yield* Deferred.succeed(abandonGate, undefined); + yield* Deferred.succeed(destroyGate, undefined); yield* Fiber.join(abandonFiber).pipe(Effect.timeout("2 seconds")); yield* Deferred.await(runtime.exit).pipe(Effect.timeout("2 seconds")); }), diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index c335d391b5..4de04a430d 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -88,8 +88,6 @@ const hostError = (operation: string, cause: unknown, reason?: "runtime-unavaila * Matches an engine CLI that is missing or reports a daemon that is not listening, never one * that rejects the caller (for example on permissions), so `destroy`/`shutdown` (`effect.ts`) * know to leave a stack registered for retry instead of surfacing a hard failure. - * Unrelated to abandonment's own cleanup-retry decision, which probes the engine directly instead - * of matching a message at all. */ const engineUnreachableAtStartup = (cause: unknown): boolean => { const message = cause instanceof Error ? cause.message : String(cause); @@ -194,7 +192,10 @@ export interface StackHostRuntime { destroy: boolean, response?: ReturnType, ) => Effect.Effect; - /** Ends ownership after a confirmed-gone registration; joins an already-running shutdown. */ + /** + * Ends ownership after a confirmed-gone registration with one best-effort destroy: it joins a + * running destroy and runs after a running stop settles. Failures are logged, never raised. + */ readonly abandon: Effect.Effect; readonly exit: Deferred.Deferred; } @@ -241,7 +242,7 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( }); const exit = yield* Deferred.make(); const gate = yield* Semaphore.make(1); - type ShutdownMode = "stop" | "destroy" | "abandon"; + type ShutdownMode = "stop" | "destroy"; const current = yield* Ref.make< { readonly mode: ShutdownMode; readonly fiber: Fiber.Fiber } | undefined >(undefined); @@ -253,17 +254,10 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( responseClosed(response).pipe(Effect.andThen(Deferred.succeed(closed, undefined))), scope, ); - // Each stage settles and logs independently: one stage's failure (for example a retained - // native command's termination that cannot be confirmed) must never skip the others. - const settled = (label: string, stage: Effect.Effect) => - stage.pipe( - Effect.catchCause((cause) => Effect.logError(`Abandoned stack: ${label} failed`, cause)), - ); - // Admits `mode`'s cleanup when nothing else is in flight: begins draining, forks the cleanup // and tracks it as `current`. Draining never reverses: a failed stop or destroy leaves its // failed fiber in `current` (repeat calls rejoin the same failure), and the owner exits - // once the response has been delivered; only abandon may replace it. + // once the response has been delivered; only abandonment may replace a settled stop. const begin = ( mode: ShutdownMode, response?: ReturnType, @@ -289,52 +283,29 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( yield* Deferred.succeed(exit, undefined); } }); - const body: Effect.Effect = - mode === "abandon" - ? Effect.gen(function* () { - // Attachment/command cleanup and namespace cleanup run concurrently; each - // retries until confirmed (or the engine is confirmed unavailable), and this - // waits for both before the owner is permitted to exit. - const commandCleanup = settled( - "command cleanup", - Owner.retryUntilConfirmed( - attachments.stopAll.pipe(Effect.andThen(runner.cleanup)), - owner.engineAvailable, - ), - ); - const namespaceCleanup = settled( - "abandon namespace cleanup", - owner.namespace.abandon, - ); - yield* Effect.all([commandCleanup, namespaceCleanup], { - concurrency: "unbounded", - discard: true, - }); - yield* finish; - }) - : Effect.gen(function* () { - yield* attachments.stopAll; - yield* runner.cleanup; - yield* mode === "stop" ? owner.namespace.stop : owner.namespace.destroy; - }).pipe( - Effect.onError(() => finish), - Effect.andThen(finish), - Effect.mapError((cause) => stackError("shutdown", cause)), - ); + const body = Effect.gen(function* () { + yield* attachments.stopAll; + yield* runner.cleanup; + yield* mode === "stop" ? owner.namespace.stop : owner.namespace.destroy; + }).pipe( + Effect.onError(() => finish), + Effect.andThen(finish), + Effect.mapError((cause) => stackError("shutdown", cause)), + ); const fiber = yield* Effect.forkIn(body, scope); yield* Ref.set(current, { mode, fiber }); return fiber; }); - // The one shutdown pipeline: `stop`, `destroy` and `abandon` are claimed - // through the same gate and `current` record, never a separate join. If a shutdown is - // already in flight when abandonment is requested, this waits for it to settle (whatever - // its outcome) and then claims abandon fresh, unless that settlement was a confirmed - // successful destroy — ownership is already fully ended, so there is nothing left to - // abandon. Once abandon is claimed, nothing else can supersede it: it always joins. + // The one shutdown pipeline: `stop` and `destroy` are claimed through the same gate and + // `current` record. A repeat request rejoins the shutdown in flight (a stop after a + // destroy, or the same mode again), and a destroy after a stop is rejected, except for + // `afterStop`: abandonment waits for the stop to settle, whatever its outcome, then claims + // its destroy fresh. const claim = ( mode: ShutdownMode, response?: ReturnType, + afterStop = false, ): Effect.Effect => Effect.gen(function* () { while (true) { @@ -346,23 +317,13 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( const fiber = yield* begin(mode, response); return { _tag: "run", fiber } as const; } - if (mode === "abandon") - return existing.mode === "abandon" - ? ({ _tag: "run", fiber: existing.fiber } as const) - : ({ - _tag: "await", - fiber: existing.fiber, - priorMode: existing.mode, - } as const); - if (existing.mode === "abandon") - return { _tag: "run", fiber: existing.fiber } as const; - if (existing.mode === "destroy" && mode === "stop") - return { _tag: "run", fiber: existing.fiber } as const; - if (existing.mode === "stop" && mode === "destroy") + if (existing.mode === "stop" && mode === "destroy") { + if (afterStop) return { _tag: "await", fiber: existing.fiber } as const; return yield* new StackError({ operation: "shutdown", message: "Shutdown mode is already selected", }); + } return { _tag: "run", fiber: existing.fiber } as const; }), ), @@ -371,12 +332,7 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( yield* Fiber.join(decision.fiber); return; } - // "await": a prior stop or destroy, whether still running or already settled, is - // never cleared for its own sake (a repeated `shutdown` call must keep rejoining it - // for free); only abandon taking over after it clears the slot, so this loop's next - // pass can claim fresh. - const outcome = yield* Fiber.join(decision.fiber).pipe(Effect.exit); - if (decision.priorMode === "destroy" && Exit.isSuccess(outcome)) return; + yield* Fiber.await(decision.fiber); yield* gate.withPermits(1)( Ref.update(current, (value) => (value?.fiber === decision.fiber ? undefined : value)), ); @@ -389,9 +345,11 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( claim(destroy ? "destroy" : "stop", response).pipe( Effect.mapError((cause) => stackError("shutdown", cause)), ); - // A registration-loss poll drives this, never an RPC caller, so there is no response to - // watch and no registration left to touch; `claim` never actually fails for this mode. - const abandon = claim("abandon").pipe(Effect.ignore); + // A registration-loss poll drives this, never an RPC caller, so there is no response to watch. + const abandon = claim("destroy", undefined, true).pipe( + Effect.tapCause((cause) => Effect.logError("Abandoned stack destroy failed", cause)), + Effect.ignore, + ); const handlers = StackRpc.of({ ...owner.handlers, runCommand: (input: RunCommandPayload) => attachments.run(input), diff --git a/packages/stack/src/runtime/Container.integration.test.ts b/packages/stack/src/runtime/Container.integration.test.ts index 9317b19c54..1863cc41ca 100644 --- a/packages/stack/src/runtime/Container.integration.test.ts +++ b/packages/stack/src/runtime/Container.integration.test.ts @@ -25,12 +25,10 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; import { HttpClient } from "effect/unstable/http"; import { - ContainerError, ContainerLaunchError, DOCKER_HOST_ALIAS, makeContainerRuntime, makeHostGateway, - probeEngineReachable, resolveEngineTarget, type ContainerProcess, type EngineTarget, @@ -1456,92 +1454,6 @@ describe("container process adapter", () => { ), ); - it.live("probes a real connection refusal to the engine as unreachable", () => - Effect.gen(function* () { - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - // A real connection attempt, not a mocked one: nothing listens on this loopback port, so - // the probe observes the docker CLI's own refusal directly, never a classified message. - const target: EngineTarget = { - engine: "docker", - argv: ["--host", "tcp://127.0.0.1:1"], - daemonId: "probe-test", - }; - expect( - yield* probeEngineReachable(spawner, target), - "a real daemon connection refusal resolves unreachable", - ).toBe(false); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("probes a real missing engine binary as unreachable", () => - Effect.gen(function* () { - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const fs = yield* FileSystem.FileSystem; - // A directory containing no `docker` binary, real `PATH` lookup included: the - // probe observes the engine's own spawn failure directly, never a stubbed spawner. - const emptyBinDir = yield* fs.makeTempDirectoryScoped({ prefix: "engine-missing-bin-" }); - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the spawned child inherits PATH; this is not application config. - const originalPath = process.env.PATH; - yield* Effect.acquireRelease( - Effect.sync(() => { - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. - process.env.PATH = emptyBinDir; - }), - () => - Effect.sync(() => { - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. - process.env.PATH = originalPath; - }), - ); - const target: EngineTarget = { engine: "docker", argv: [], daemonId: "probe-test" }; - expect( - yield* probeEngineReachable(spawner, target), - "a real spawn ENOENT for the missing docker binary resolves unreachable", - ).toBe(false); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live( - "keeps retrying a daemon-relayed failure, such as a registry pull refusal, while the real engine stays reachable", - () => - Effect.gen(function* () { - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const target = yield* resolveEngineTarget(spawner, testEngine); - const attempts = yield* Ref.make(0); - const thirdAttemptObserved = yield* Deferred.make(); - // Simulates a registry rejecting a pull while the daemon itself answers normally: this - // must never be mistaken for an unavailable engine, so `retryUntilConfirmed` keeps - // retrying for as long as the probe keeps confirming the daemon is reachable. A third - // attempt is this test's own observed signal, so a slow individual probe cannot flake it. - const daemonRelayedFailure = Ref.updateAndGet(attempts, (count) => count + 1).pipe( - Effect.tap((count) => - count >= 3 ? Deferred.succeed(thirdAttemptObserved, undefined) : Effect.void, - ), - Effect.andThen( - Effect.fail( - new ContainerError({ - operation: "pull", - message: - "Error response from daemon: pull access denied for supabase/does-not-exist-image, repository does not exist or may require 'docker login'", - }), - ), - ), - ); - // Backoff between attempts is this test's only controlled time; the daemon probe - // itself stays real, so the attempts it takes to observe a third one are never rushed. - const retrying = yield* Effect.forkDetach( - Owner.retryUntilConfirmed( - daemonRelayedFailure, - probeEngineReachable(spawner, target), - ).pipe(Effect.provideService(Owner.RetryUntilConfirmedInitialDelay, "1 millis")), - ); - // A guard against a genuine hang, not a budget this test's attempt count is sampled from. - yield* Deferred.await(thirdAttemptObserved).pipe(Effect.timeout("30 seconds")); - yield* Fiber.interrupt(retrying); - expect(yield* Ref.get(attempts)).toBeGreaterThanOrEqual(3); - }).pipe(Effect.provide(NodeServices.layer)), - ); - it.live( "keeps helper creation, runtime launches, password reconcile and reconcile pinned to the context an owner resolved at startup, despite a later context switch", () => diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index c85fc0556d..ab117acd0a 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -281,27 +281,6 @@ export const resolveEngineTarget = Effect.fn("Container.resolveEngineTarget")(fu /** A pull worth retrying: rate-limited or a dropped connection. */ const retryablePull = (error: ContainerError) => rateLimited(error) || transientPullFailure(error); -/** - * Probes the pinned engine target directly, with the cheapest real call (`info`), rather than - * classifying a failure's message: a spawn failure (missing binary) or the daemon - * endpoint refusing the connection both resolve `false`; any other failure (a daemon-relayed - * error, for example a registry rejecting a pull while the daemon itself is up) never reaches - * this probe and so never resolves `false` through it. - */ -export const probeEngineReachable = ( - spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], - target: EngineTarget, -): Effect.Effect => - runRaw(spawner, target.engine, [ - ...target.argv, - "info", - "--format", - target.engine === "docker" ? "{{.ID}}" : "{{.Host.Hostname}}", - ]).pipe( - Effect.map((id) => id.trim().length > 0), - Effect.orElseSucceed(() => false), - ); - const PULL_MAX_RETRIES = 4; /** @@ -1157,7 +1136,7 @@ const listStackContainers = Effect.fn("Container.listStackContainers")(function* /** * Removes every container carrying this stack's identity and data-root labels and reports how - * many remain (normally 0): the registration-independent confirming sweep abandonment uses + * many remain (normally 0): the registration-independent confirming sweep destroy and stop use * instead of depending on any helper registry's own bookkeeping. A container that disappears * between listing and removal is not an error, matching {@link removeContainerById}. */ From a33ace9a32883fff63835c69e72dfd7ad84705ab Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 19:32:15 +0200 Subject: [PATCH 113/142] fix(stack): keep a crash outcome through a failed cleanup --- packages/stack/src/Lifecycle.ts | 7 ++++- packages/stack/src/Lifecycle.unit.test.ts | 31 +++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/packages/stack/src/Lifecycle.ts b/packages/stack/src/Lifecycle.ts index 575653449f..6ec86c8cfc 100644 --- a/packages/stack/src/Lifecycle.ts +++ b/packages/stack/src/Lifecycle.ts @@ -784,7 +784,12 @@ const applyEvent = ( return [ setService(dependentsFailed, id, (s) => ({ ...failAllWaiters(s, id, message, cause, commands), - phase: Phase.Stopping({ generation }), + phase: Phase.Stopping({ + generation, + ...(service.phase._tag === "Stopping" && service.phase.crash !== undefined + ? { crash: service.phase.crash } + : {}), + }), breaker: failure === undefined || service.phase._tag === "Stopping" || diff --git a/packages/stack/src/Lifecycle.unit.test.ts b/packages/stack/src/Lifecycle.unit.test.ts index de6c97be45..85f9123f68 100644 --- a/packages/stack/src/Lifecycle.unit.test.ts +++ b/packages/stack/src/Lifecycle.unit.test.ts @@ -748,6 +748,37 @@ describe("Lifecycle reducer", () => { expect(state.services.get("api")?.breaker.consecutiveFailures).toBe(1); }); + it("ends Failed without relaunching an eager service whose crash cleanup failed and was retried by traffic", () => { + let state = initialState( + makeGraph([{ id: "database", activation: "eager", prerequisites: [] }]), + ); + ({ state } = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "database" }), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "database", generation: 1 }), now: 1 }, + sessionLost("database", 1, 2), + { + event: LifecycleEvent.StopFailed({ id: "database", generation: 1, cause: "busy" }), + now: 3, + }, + ])); + + let commandsByStep: ReadonlyArray>; + ({ state, commandsByStep } = run(state, [ + { event: open("database", 1), now: 4 }, + { event: LifecycleEvent.WaiterCancelled({ id: "database", waiterId: 1 }), now: 5 }, + { + event: LifecycleEvent.Exited({ id: "database", generation: 1, requested: true }), + now: 6, + }, + ])); + expect(tagsOf(commandsByStep[0])).toContain("Stop"); + expect(commandsByStep[2]).toEqual([]); + expect(state.services.get("database")?.phase).toMatchObject({ + _tag: "Failed", + cause: "crash", + }); + }); + it("ends Stopped when an explicit stop arrives during the crash's cleanup", () => { let state = initialState(makeGraph([lazy("api")])); ({ state } = run(state, [ From 99e3eb8456d5494d525da1afa763b590db7a2ada Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 6 Oct 2026 19:32:17 +0200 Subject: [PATCH 114/142] fix(stack): confirm a vanished registration when destroy cannot take the registry lock --- packages/stack/src/Owner.ts | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index c81806dbe8..dd4752d50e 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -282,27 +282,30 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { options.state.withLock( readSaved.pipe(Effect.flatMap((current) => options.state.save(update(current)))), ); - // Idempotent on an already-missing registration (a deleted or already-destroyed state.json), - // unlike `updateState`: destroy must still be able to confirm there is nothing left to publish - // when the registration, or the whole state root with its registry lock, vanished from under it. + // Idempotent on an already-missing registration: the registration, or the whole state root + // with its registry lock, may vanish under destroy, so a failure counts only while it remains. const removeInstanceRegistration = (id: string) => options.state - .read(stackId) + .withLock( + options.state + .read(stackId) + .pipe( + Effect.flatMap((current) => + current === undefined + ? Effect.void + : options.state.save(withoutInstance(current, id)), + ), + ), + ) .pipe( - Effect.flatMap((registered) => - registered === undefined - ? Effect.void - : options.state.withLock( - options.state - .read(stackId) - .pipe( - Effect.flatMap((current) => - current === undefined - ? Effect.void - : options.state.save(withoutInstance(current, id)), - ), - ), + Effect.catch((error) => + options.state + .read(stackId) + .pipe( + Effect.flatMap((registered) => + registered === undefined ? Effect.void : Effect.fail(error), ), + ), ), ); From aa0da1c7c30f64fb5caa9ed4c544c64465f62c9d Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 09:00:12 +0200 Subject: [PATCH 115/142] refactor(stack): drop status fields no consumer reads Remove launchId, cleanupError and destroyPending from the service status and the RPC observation. Tests now observe relaunches, cleanup failures and queued destroys through lifecycle, health, error and their runtime fixtures. --- .../db-config.integration.test.ts | 2 - .../commands/db/dump/dump.integration.test.ts | 2 - .../db/reset/reset.integration.test.ts | 2 - .../generate/generate.integration.test.ts | 2 - .../declarative/sync/sync.integration.test.ts | 2 - .../db/start/start.integration.test.ts | 2 - .../stack/prepare/prepare.integration.test.ts | 2 - .../stack/restart/restart.integration.test.ts | 34 ++++++++-- .../start-export-pointer.integration.test.ts | 2 - .../stack/start/start.integration.test.ts | 2 - .../stack/status/status.integration.test.ts | 2 - .../functions/serve/serve.stack.e2e.test.ts | 40 +++++++++-- .../serve/serve.stack.integration.test.ts | 2 - apps/cli/tests/helpers/storage.ts | 2 - .../src/Orchestrator.integration.test.ts | 57 ++++++++-------- packages/stack/src/Orchestrator.ts | 14 ---- packages/stack/src/Rpc.ts | 2 - .../stack/src/Service.integration.test.ts | 68 ++++++++++++------- .../ProcessRecipe.integration.test.ts | 13 ++-- packages/stack/tests/whole-stack/idle.ts | 7 +- 20 files changed, 146 insertions(+), 113 deletions(-) diff --git a/apps/cli/src/command-internal/db-config.integration.test.ts b/apps/cli/src/command-internal/db-config.integration.test.ts index 1ce21541bf..5a15d55905 100644 --- a/apps/cli/src/command-internal/db-config.integration.test.ts +++ b/apps/cli/src/command-internal/db-config.integration.test.ts @@ -456,10 +456,8 @@ describe("dbConfigResolver (db-url under the stack backend)", () => { lifecycle: state === "running" ? "running" : "stopped", health: state === "running" ? "healthy" : undefined, error: undefined, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, wakeEnabled: state === "running", }), followStatus: Stream.empty, diff --git a/apps/cli/src/commands/db/dump/dump.integration.test.ts b/apps/cli/src/commands/db/dump/dump.integration.test.ts index d7f0a8c4f7..bf806853ea 100644 --- a/apps/cli/src/commands/db/dump/dump.integration.test.ts +++ b/apps/cli/src/commands/db/dump/dump.integration.test.ts @@ -115,10 +115,8 @@ const managedDumpStackApi = (runtime: "native" | "docker") => { lifecycle: "running", health: "healthy", error: undefined, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, wakeEnabled: true, }), followStatus: Stream.empty, diff --git a/apps/cli/src/commands/db/reset/reset.integration.test.ts b/apps/cli/src/commands/db/reset/reset.integration.test.ts index 4df92b184a..5d65436cb5 100644 --- a/apps/cli/src/commands/db/reset/reset.integration.test.ts +++ b/apps/cli/src/commands/db/reset/reset.integration.test.ts @@ -614,10 +614,8 @@ function makeStackObservation( lifecycle: opts.lifecycle ?? "stopped", health: opts.health, error: opts.error, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, wakeEnabled: opts.wakeEnabled ?? false, }; } diff --git a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts index 26f53e380a..7a6ba5a7ab 100644 --- a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts @@ -130,10 +130,8 @@ function generateStackApi(workdir: string) { health: "healthy", wakeEnabled: true, currentOperation: undefined, - launchId: undefined, exit: undefined, error: undefined, - cleanupError: undefined, }), }; const composition = { diff --git a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts index aaa03ce522..3db886304a 100644 --- a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts @@ -132,10 +132,8 @@ function syncStackApi(workdir: string, port: number) { health: "healthy", wakeEnabled: true, currentOperation: undefined, - launchId: undefined, exit: undefined, error: undefined, - cleanupError: undefined, }), }; const composition = { diff --git a/apps/cli/src/commands/db/start/start.integration.test.ts b/apps/cli/src/commands/db/start/start.integration.test.ts index ccb0615fa2..d41d108b88 100644 --- a/apps/cli/src/commands/db/start/start.integration.test.ts +++ b/apps/cli/src/commands/db/start/start.integration.test.ts @@ -1624,10 +1624,8 @@ describe("db start stack backend", () => { lifecycle: state.running ? ("running" as const) : ("stopped" as const), health: state.running ? ("healthy" as const) : undefined, error: undefined, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, wakeEnabled: state.running, })), followStatus: Stream.empty, diff --git a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts index 7b0192e2f7..280025b59b 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts @@ -104,10 +104,8 @@ const makeFixture = (root: string, options: FixtureOptions = {}) => { lifecycle: "stopped" as const, health: undefined, error: undefined, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, wakeEnabled: false, }), followStatus: Stream.empty, diff --git a/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts b/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts index ab24e7622a..e848d47b84 100644 --- a/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts @@ -2,7 +2,7 @@ import { tmpdir } from "node:os"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { StackError } from "@supabase/stack/effect"; -import { Effect, FileSystem, Layer, Option, Path } from "effect"; +import { Deferred, Effect, Fiber, FileSystem, Layer, Option, Path, Ref, Stream } from "effect"; import { mockOutput } from "../../../../../tests/helpers/mocks.ts"; import { mockCommandSettings, @@ -71,17 +71,43 @@ describe("stack restart", () => { yield* standalone.start; yield* standalone.ready; const before = yield* member.status; - const standaloneBefore = yield* standalone.status; + + // Both follows are subscribed once their first status arrives, before the restart. + const memberFollowing = yield* Deferred.make(); + const relaunched = yield* member.followStatus.pipe( + Stream.tap(() => Deferred.succeed(memberFollowing, undefined)), + Stream.dropWhile((value) => value.lifecycle === "running"), + Stream.filter( + (value) => value.lifecycle === "running" && value.health === "healthy", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + const standaloneFollowing = yield* Deferred.make(); + const standaloneLifecycles = yield* Ref.make>([]); + yield* standalone.followStatus.pipe( + Stream.tap((value) => + Ref.update(standaloneLifecycles, (seen) => [...seen, value.lifecycle]).pipe( + Effect.andThen(Deferred.succeed(standaloneFollowing, undefined)), + ), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* Deferred.await(memberFollowing); + yield* Deferred.await(standaloneFollowing); + const result = yield* stackRestart(f.flags).pipe(Effect.provide(f.layer)); + yield* Fiber.join(relaunched); const after = yield* member.status; const standaloneAfter = yield* standalone.status; expect(result.map(({ id }) => id)).toEqual([member.id]); expect(after.lifecycle).toBe("running"); expect(after.health).toBe("healthy"); - expect(after.launchId).not.toBe(before.launchId); expect(after.endpoints).toEqual(before.endpoints); expect(standaloneAfter.lifecycle).toBe("running"); - expect(standaloneAfter.launchId).toBe(standaloneBefore.launchId); + expect(new Set(yield* Ref.get(standaloneLifecycles))).toEqual(new Set(["running"])); expect(f.output.stdoutText).toContain("using its saved configuration"); expect(f.telemetry.flushed).toBe(true); yield* stack.stop; diff --git a/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts index 7ae6e0f6b6..05b6e6e648 100644 --- a/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts @@ -65,10 +65,8 @@ function makeDatabaseStack(sqlPort: number, credentials: StackCredentials): Stac lifecycle: "running", health: "healthy", error: undefined, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, wakeEnabled: false, }; const databaseInstance = { diff --git a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts index b15f1c642b..296d3ed8d8 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts @@ -107,10 +107,8 @@ const instance = ( lifecycle: view().lifecycle, health: view().health, error: undefined, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, wakeEnabled: view().wakeEnabled, }); const base = { diff --git a/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts b/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts index 3396d83a97..5fa6b4c87e 100644 --- a/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts @@ -102,10 +102,8 @@ const makeObservation = ( lifecycle: "stopped", health: undefined, error: undefined, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, wakeEnabled: true, ...input, }); diff --git a/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts b/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts index 5bc41babe2..a860a48d3e 100644 --- a/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts +++ b/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts @@ -1,7 +1,18 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { create as createStack } from "@supabase/stack/effect"; -import { Data, Effect, FileSystem, Layer, Path, Redacted, Schema } from "effect"; +import { + Data, + Deferred, + Effect, + FileSystem, + Layer, + Path, + Redacted, + Ref, + Schema, + Stream, +} from "effect"; import { FetchHttpClient, HttpClient } from "effect/unstable/http"; import { homedir, tmpdir } from "node:os"; @@ -177,6 +188,25 @@ const payload = Effect.fn("FunctionsServeE2e.payload")(function* ( const layer = Layer.mergeAll(BunServices.layer, FetchHttpClient.layer); +/** Records every lifecycle the service reports once its follow is subscribed. */ +const trackLifecycles = Effect.fn("FunctionsServeE2e.trackLifecycles")(function* (instance: { + readonly followStatus: Stream.Stream<{ readonly lifecycle: string }, unknown>; +}) { + const seen = yield* Ref.make>(new Set()); + const subscribed = yield* Deferred.make(); + yield* instance.followStatus.pipe( + Stream.tap((value) => + Ref.update(seen, (lifecycles) => new Set([...lifecycles, value.lifecycle])).pipe( + Effect.andThen(Deferred.succeed(subscribed, undefined)), + ), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* Deferred.await(subscribed); + return seen; +}); + describe("functions serve (stack e2e)", () => { for (const runtime of ["native", "docker"] as const) { if (runtime === "native" && !nativeSupported) continue; @@ -195,7 +225,7 @@ describe("functions serve (stack e2e)", () => { expect(saved.wakeEnabled).toBe(true); const composition = yield* stack.composition.describe; const plain = yield* serve(root, home); - const active = yield* functions.status; + const lifecycles = yield* trackLifecycles(functions); expect(yield* payload(apiUrl, true)).toEqual({ value: "original", anon: true, @@ -212,7 +242,7 @@ describe("functions serve (stack e2e)", () => { expect((yield* payload(apiUrl, true)).value).toBeNull(); yield* fs.writeFileString(sourcePath, source); yield* interrupt(plain); - expect((yield* functions.status).launchId).toBe(active.launchId); + expect(yield* Ref.get(lifecycles)).toEqual(new Set(["running"])); expect((yield* invoke(apiUrl, false)).status).toBe(401); const overridden = yield* serve(root, home, [ @@ -246,7 +276,7 @@ describe("functions serve (stack e2e)", () => { const composition = yield* stack.composition.describe; const database = services.find((instance) => instance.service === "database"); if (database === undefined) return yield* Effect.die("Database missing"); - const databaseLaunch = (yield* database.status).launchId; + const databaseLifecycles = yield* trackLifecycles(database); const child = yield* serve(root, home); expect(yield* payload(apiUrl, true)).toEqual({ value: null, @@ -261,7 +291,7 @@ describe("functions serve (stack e2e)", () => { services.map(({ id }) => id).sort(), ); expect(yield* stack.composition.describe).toEqual(composition); - expect((yield* database.status).launchId).toBe(databaseLaunch); + expect(yield* Ref.get(databaseLifecycles)).toEqual(new Set(["running"])); expect((yield* invoke(apiUrl, false)).status).toBe(404); const http = yield* HttpClient.HttpClient; expect((yield* http.get(`${apiUrl}/rest/v1/`)).status).toBe(200); diff --git a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts index f1c68f27dc..9b144b067c 100644 --- a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts +++ b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts @@ -90,9 +90,7 @@ const observation = ( health: "healthy", currentOperation: undefined, error: undefined, - cleanupError: undefined, exit: undefined, - launchId: 1, wakeEnabled: false, ...overrides, }); diff --git a/apps/cli/tests/helpers/storage.ts b/apps/cli/tests/helpers/storage.ts index 654b6eb7dc..d7f93d2549 100644 --- a/apps/cli/tests/helpers/storage.ts +++ b/apps/cli/tests/helpers/storage.ts @@ -156,10 +156,8 @@ const observation = ( health, error: error === undefined ? undefined : { _tag: "ServiceError", operation: "status", message: error }, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, wakeEnabled, }); diff --git a/packages/stack/src/Orchestrator.integration.test.ts b/packages/stack/src/Orchestrator.integration.test.ts index d36d4dcac6..e2dd01fab3 100644 --- a/packages/stack/src/Orchestrator.integration.test.ts +++ b/packages/stack/src/Orchestrator.integration.test.ts @@ -21,6 +21,7 @@ import * as TestClock from "effect/testing/TestClock"; import { HttpClient } from "effect/unstable/http"; import { createServer } from "node:http"; // oxlint-disable-line effecttsgo/node-builtin-import -- real backend behind the listener. import { captureLogs } from "../tests/logs.ts"; +import type { LifecycleEvent } from "./Lifecycle.ts"; import * as Network from "./Network.ts"; import * as Orchestrator from "./Orchestrator.ts"; import type { RegisteredInstance } from "./Orchestrator.ts"; @@ -49,6 +50,8 @@ const makeInstance = ( /** Every launch's runtime exit, for workloads that crash on demand. */ readonly crash?: Effect.Effect>; readonly events?: Ref.Ref>; + /** Runs after the lifecycle has applied each event the service reports. */ + readonly onReport?: (event: LifecycleEvent) => Effect.Effect; } = {}, ) => Effect.gen(function* () { @@ -86,7 +89,14 @@ const makeInstance = ( }), removeData: () => event("destroy").pipe(Effect.andThen(options.removeData ?? Effect.void)), }, - { id, config: {}, report: orchestrator.report }, + { + id, + config: {}, + report: (reported) => + orchestrator + .report(reported) + .pipe(Effect.andThen(options.onReport?.(reported) ?? Effect.void)), + }, ); const configFor = (inputs: Record, candidate: unknown) => candidate === undefined @@ -727,7 +737,6 @@ describe("service composition", () => { }); yield* orchestrator.startComposition; yield* Effect.scoped(orchestrator.acquire("rest")); - const firstLaunch = (yield* rest.status).launchId; yield* Deferred.succeed( crashed, @@ -748,9 +757,7 @@ describe("service composition", () => { yield* Deferred.succeed(cleanupGate, undefined); yield* Fiber.join(waking); expect(yield* Ref.get(rest.starts)).toHaveLength(2); - const status = yield* rest.status; - expect(status.lifecycle).toBe("running"); - expect(status.launchId).not.toBe(firstLaunch); + expect((yield* rest.status).lifecycle).toBe("running"); yield* orchestrator.stopNamespace; }), ).pipe(Effect.provide(TestClock.layer())), @@ -1503,7 +1510,7 @@ describe("idle races through a real listener", () => { .pipe(Effect.forkChild({ startImmediately: true })); yield* Deferred.await(stack.holdEntered); yield* TestClock.adjust("10 seconds"); - expect(yield* stack.status).toMatchObject({ lifecycle: "running", launchId: 1 }); + expect((yield* stack.status).lifecycle).toBe("running"); yield* stack.releaseHeld; expect(yield* Fiber.join(request)).toEqual({ status: 200, body: "generation:1" }); @@ -1537,7 +1544,7 @@ describe("idle races through a real listener", () => { yield* Deferred.succeed(gate.release, undefined); expect(yield* Fiber.join(request)).toEqual({ status: 200, body: "generation:2" }); - expect(yield* stack.status).toMatchObject({ lifecycle: "running", launchId: 2 }); + expect((yield* stack.status).lifecycle).toBe("running"); yield* stack.orchestrator.stopNamespace; }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, TestClock.layer()))), @@ -1731,7 +1738,10 @@ describe("failed and crashed services", () => { const orchestrator = yield* makeTestOrchestrator(); const crashes = yield* Queue.unbounded>(); const stops = yield* Ref.make(0); + const cleanupFailures = yield* Queue.unbounded(); const api = yield* makeInstance(orchestrator, "api", { + onReport: (event) => + event._tag === "StopFailed" ? Queue.offer(cleanupFailures, undefined) : Effect.void, crash: Queue.take(crashes), // Every generation's first cleanup attempt fails; the retry succeeds. stop: Ref.updateAndGet(stops, (count) => count + 1).pipe( @@ -1747,19 +1757,9 @@ describe("failed and crashed services", () => { yield* orchestrator.startComposition; const serveThenCrash = Effect.gen(function* () { yield* Effect.scoped(orchestrator.acquire("api")); - const cleanupPending = yield* api.observation.pipe( - Stream.filter( - (state) => - state.lifecycle === "stopping" && - state.currentOperation === undefined && - state.cleanupError !== undefined, - ), - Stream.take(1), - Stream.runDrain, - Effect.forkChild({ startImmediately: true }), - ); yield* Queue.offer(crashes, Exit.fail(failure("segfault"))); - yield* Fiber.join(cleanupPending); + yield* Queue.take(cleanupFailures); + expect((yield* api.status).lifecycle).toBe("stopping"); }); yield* serveThenCrash; @@ -1777,7 +1777,12 @@ describe("failed and crashed services", () => { Effect.gen(function* () { const orchestrator = yield* makeTestOrchestrator(); const crash = yield* Deferred.make>(); - const database = yield* makeInstance(orchestrator, "database", { exit: crash }); + const launched = yield* Queue.unbounded(); + const database = yield* makeInstance(orchestrator, "database", { + exit: crash, + onReport: (event) => + event._tag === "LaunchSucceeded" ? Queue.offer(launched, undefined) : Effect.void, + }); const rest = yield* makeInstance(orchestrator, "rest"); yield* orchestrator.configure({ members: [ @@ -1788,17 +1793,13 @@ describe("failed and crashed services", () => { }); yield* orchestrator.startComposition; yield* Effect.scoped(orchestrator.acquire("rest")); + yield* Queue.take(launched); - const relaunched = yield* database.observation.pipe( - Stream.filter((state) => state.launchId === 2 && state.health === "healthy"), - Stream.take(1), - Stream.runDrain, - Effect.forkChild({ startImmediately: true }), - ); yield* Deferred.succeed(crash, Exit.fail(failure("segfault"))); - yield* Fiber.join(relaunched); + yield* Queue.take(launched); - expect(yield* rest.status).toMatchObject({ lifecycle: "running", launchId: 1 }); + expect((yield* rest.status).lifecycle).toBe("running"); + expect(yield* Ref.get(database.starts)).toHaveLength(2); expect(yield* Ref.get(rest.starts)).toHaveLength(1); yield* Effect.scoped(orchestrator.acquire("rest")); yield* orchestrator.stopNamespace; diff --git a/packages/stack/src/Orchestrator.ts b/packages/stack/src/Orchestrator.ts index 721e1fdf20..92f04c5d24 100644 --- a/packages/stack/src/Orchestrator.ts +++ b/packages/stack/src/Orchestrator.ts @@ -140,13 +140,8 @@ const waiterBudgetMillis = 120_000; export interface Status extends ServiceObservation { readonly lifecycle: "stopped" | "starting" | "running" | "stopping"; readonly health: "starting" | "healthy" | "unhealthy" | undefined; - readonly cleanupError: ServiceError | undefined; - /** The live generation, if any. */ - readonly launchId: number | undefined; /** Whether demand (traffic, a dependent or eager intent) relaunches the service. */ readonly wakeEnabled: boolean; - /** Whether a destroy is waiting for the service to stop and its storage to be free, or removing it. */ - readonly destroyPending: boolean; } /** @@ -355,16 +350,7 @@ const statusOf = ( ...execution, lifecycle: lifecycleOf(state, id), health, - cleanupError: - service?.cleanupFailure === undefined - ? undefined - : cleanupError(id, service.cleanupFailure.cause), - launchId: - phase === undefined || phase._tag === "Stopped" || phase._tag === "Failed" - ? undefined - : phase.generation, wakeEnabled: wakeEnabledOf(state, id), - destroyPending: service?.destroy !== undefined, }; }; diff --git a/packages/stack/src/Rpc.ts b/packages/stack/src/Rpc.ts index 0c57dcc1d3..1cac14e545 100644 --- a/packages/stack/src/Rpc.ts +++ b/packages/stack/src/Rpc.ts @@ -123,12 +123,10 @@ export const Observation = Schema.Struct({ lifecycle: Schema.Literals(["stopped", "starting", "running", "stopping"]), health: Schema.UndefinedOr(Schema.Literals(["starting", "healthy", "unhealthy"])), error: Schema.UndefinedOr(ServiceErrorSchema), - cleanupError: Schema.UndefinedOr(ServiceErrorSchema), exit: Schema.UndefinedOr(Schema.Exit(Schema.Void, ServiceErrorSchema, Schema.Defect())), currentOperation: Schema.UndefinedOr( Schema.Literals(["start", "stop", "restart", "storage", "destroy", "sleep"]), ), - launchId: Schema.UndefinedOr(Schema.Int), wakeEnabled: Schema.Boolean, }); export interface Observation extends Schema.Schema.Type {} diff --git a/packages/stack/src/Service.integration.test.ts b/packages/stack/src/Service.integration.test.ts index 0bd6a45a07..fdab67ce9c 100644 --- a/packages/stack/src/Service.integration.test.ts +++ b/packages/stack/src/Service.integration.test.ts @@ -172,12 +172,27 @@ const partialDefinition = (plans: Queue.Queue): ServiceDefinition - service.observation.pipe( - Stream.filter((status) => status.destroyPending), - Stream.runHead, - ); +/** + * Resolves once the next change after the current status is published. The subscription is live + * when this returns, so a destroy queued behind a held storage operation, the only change in that + * window, can't be missed. + */ +const awaitNextChange = (service: Standalone) => + Effect.gen(function* () { + const subscribed = yield* Deferred.make(); + const change = yield* service.observation.pipe( + Stream.zipWithIndex, + Stream.tap(([, index]) => + index === 0 ? Deferred.succeed(subscribed, undefined) : Effect.void, + ), + Stream.drop(1), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* Deferred.await(subscribed); + return change; + }); const stopFixture = (service: Standalone, plan: RuntimePlan) => Effect.gen(function* () { @@ -298,11 +313,11 @@ describe("service execution", () => { yield* Fiber.join(stopAdmitted); yield* open(staleGate); yield* Fiber.join(restarted); - expect((yield* service.get).launchId).toBe(2); // The earlier start is satisfied by the restarted generation. expect(Exit.isSuccess(yield* Fiber.join(stale))).toBe(true); expect(yield* service.get).toMatchObject({ lifecycle: "running", error: undefined }); + expect((yield* Ref.get(plan.state)).phase).toBe("launched"); yield* stopFixture(service, plan); }), ), @@ -451,7 +466,6 @@ describe("service execution", () => { expect(Exit.isFailure(yield* partial.service.start.pipe(Effect.exit))).toBe(true); expect(yield* partial.service.get).toMatchObject({ lifecycle: "stopping", error: failure }); - expect((yield* partial.service.get).cleanupError).toBeDefined(); expect((yield* Ref.get(plan.state)).removed).toBe(false); const retry = yield* partial.service.stop.pipe(Effect.forkScoped); @@ -460,10 +474,7 @@ describe("service execution", () => { yield* Deferred.await(plan.removeStarted); yield* open(plan.removeGate); yield* Fiber.join(retry); - expect(yield* partial.service.get).toMatchObject({ - lifecycle: "stopped", - cleanupError: undefined, - }); + expect((yield* partial.service.get).lifecycle).toBe("stopped"); expect((yield* Ref.get(plan.state)).removed).toBe(true); const cleaned = yield* makeFixture("database-partial-clean", partialDefinition); @@ -616,19 +627,18 @@ describe("service execution", () => { yield* open(plan.stopGate); yield* Ref.set(plan.removeFailure, true); expect(Exit.isFailure(yield* fixture.service.stop.pipe(Effect.exit))).toBe(true); - expect(yield* fixture.service.get).toMatchObject({ lifecycle: "stopping" }); - expect((yield* fixture.service.get).cleanupError).toBeDefined(); + expect(yield* fixture.service.get).toMatchObject({ + lifecycle: "stopping", + error: { operation: "remove" }, + }); expect((yield* Ref.get(plan.state)).removed).toBe(false); const retry = yield* fixture.service.stop.pipe(Effect.forkScoped); yield* Deferred.await(plan.removeStarted); - expect((yield* fixture.service.get).cleanupError).toBeDefined(); + expect((yield* fixture.service.get).lifecycle).toBe("stopping"); yield* open(plan.removeGate); yield* Fiber.join(retry); - expect(yield* fixture.service.get).toMatchObject({ - lifecycle: "stopped", - cleanupError: undefined, - }); + expect((yield* fixture.service.get).lifecycle).toBe("stopped"); expect((yield* Ref.get(plan.state)).removed).toBe(true); }), ), @@ -726,8 +736,9 @@ describe("service execution", () => { yield* Deferred.await(storageStarted); expect((yield* service.get).currentOperation).toBe("storage"); + const destroyRequested = yield* awaitNextChange(service); const destroyed = yield* service.destroy.pipe(Effect.forkScoped); - yield* awaitDestroyPending(service); + yield* Fiber.join(destroyRequested); expect(yield* service.listed).toBe(true); yield* open(storageGate); yield* Fiber.join(storage); @@ -758,11 +769,11 @@ describe("service execution", () => { .pipe(Effect.forkScoped); yield* Deferred.await(storageStarted); + const destroyRequested = yield* awaitNextChange(service); const destroyed = yield* service.destroy.pipe(Effect.forkScoped); - yield* awaitDestroyPending(service); + yield* Fiber.join(destroyRequested); yield* Fiber.interrupt(destroyed); expect(Exit.hasInterrupts(yield* Fiber.await(destroyed))).toBe(true); - expect((yield* service.get).destroyPending).toBe(false); yield* open(storageGate); yield* Fiber.join(storage); @@ -869,7 +880,11 @@ describe("service execution", () => { const newReady = yield* fixture.service.ready.pipe(Effect.forkScoped); yield* open(second.healthGate); expect(Exit.isSuccess(yield* Fiber.await(newReady))).toBe(true); - expect((yield* fixture.service.get).launchId).toBe(2); + expect(yield* fixture.service.get).toMatchObject({ + lifecycle: "running", + health: "healthy", + }); + expect((yield* Ref.get(second.state)).health).toBe("healthy"); yield* stopFixture(fixture.service, second); }), ), @@ -1018,7 +1033,7 @@ describe("service readiness recovery", () => { const firstLaunchChecks = yield* Ref.make(0); const probeStarted = yield* Deferred.make(); const probeInterrupted = yield* Deferred.make(); - const { service } = yield* makeProbedService((launch) => + const { service, launches } = yield* makeProbedService((launch) => launch === 1 ? Ref.updateAndGet(firstLaunchChecks, (count) => count + 1).pipe( Effect.flatMap((count) => @@ -1040,7 +1055,8 @@ describe("service readiness recovery", () => { const relaunched = yield* waitForStatus( service, - (value) => value.launchId === 2 && value.health === "unhealthy", + (value) => + value.health === "unhealthy" && value.error?.message === "second launch unhealthy", ); yield* service.restart(); yield* Fiber.join(relaunched); @@ -1048,10 +1064,10 @@ describe("service readiness recovery", () => { yield* Deferred.await(probeInterrupted); expect(Exit.isFailure(yield* Fiber.join(stale))).toBe(true); expect(yield* service.get).toMatchObject({ - launchId: 2, health: "unhealthy", error: { message: "second launch unhealthy" }, }); + expect(yield* Ref.get(launches)).toBe(2); expect((yield* Effect.flip(service.ready)).message).toContain("second launch unhealthy"); expect(yield* Ref.get(firstLaunchChecks)).toBe(2); yield* service.stop; diff --git a/packages/stack/src/services/ProcessRecipe.integration.test.ts b/packages/stack/src/services/ProcessRecipe.integration.test.ts index 3642f6f2a2..7af2c1acec 100644 --- a/packages/stack/src/services/ProcessRecipe.integration.test.ts +++ b/packages/stack/src/services/ProcessRecipe.integration.test.ts @@ -200,8 +200,10 @@ describe("ProcessRecipe launch cleanup", () => { }); expect(Exit.isFailure(yield* service.start.pipe(Effect.exit))).toBe(true); - expect((yield* service.get).lifecycle).toBe("stopping"); - expect((yield* service.get).cleanupError?.operation).toBe("stop"); + expect(yield* service.get).toMatchObject({ + lifecycle: "stopping", + error: { operation: "launch" }, + }); expect(yield* Ref.get(startupLaunches)).toBe(1); expect(yield* Ref.get(serviceLaunches)).toBe(0); expect(yield* Ref.get(active)).toBe(1); @@ -289,14 +291,15 @@ describe("ProcessRecipe launch cleanup", () => { const failure = yield* service.start.pipe(Effect.exit); expect(Exit.isFailure(failure)).toBe(true); - expect((yield* service.get).lifecycle).toBe("stopping"); - expect((yield* service.get).cleanupError?.operation).toBe("stop"); + expect(yield* service.get).toMatchObject({ + lifecycle: "stopping", + error: { operation: "launch" }, + }); expect(startupWrapped).toBe(true); yield* Ref.set(cleanupFailure, false); yield* service.stop; expect((yield* service.get).lifecycle).toBe("stopped"); - expect((yield* service.get).cleanupError).toBeUndefined(); }).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ), ); diff --git a/packages/stack/tests/whole-stack/idle.ts b/packages/stack/tests/whole-stack/idle.ts index 221fcf9c6e..e8bdb13e4c 100644 --- a/packages/stack/tests/whole-stack/idle.ts +++ b/packages/stack/tests/whole-stack/idle.ts @@ -235,8 +235,7 @@ export const restartIdleWake = (runtime: Runtime) => const rewarm = yield* requestWithHeaders(restUrl, headers); expect(rewarm.status, rewarm.body).toBe(200); - const awake = yield* rest.status; - expect(awake.lifecycle).toBe("running"); + expect((yield* rest.status).lifecycle).toBe("running"); yield* killWorkload(runtime, fixture, rest.id); const crashed = yield* rest.followStatus.pipe( @@ -249,9 +248,7 @@ export const restartIdleWake = (runtime: Runtime) => const rewake = yield* requestWithHeaders(restUrl, headers); expect(rewake.status, rewake.body).toBe(200); - const woken = yield* rest.status; - expect(woken.lifecycle).toBe("running"); - expect(woken.launchId).toBeGreaterThan(awake.launchId ?? Number.POSITIVE_INFINITY); + expect((yield* rest.status).lifecycle).toBe("running"); }), ); From c4a15940a98378571c701dc09c9007ba62c491a9 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 09:17:59 +0200 Subject: [PATCH 116/142] test(stack): keep shared artifact caches out of OS-aged temp directories macOS ages out files under the temp directories by access time, which left published native generations missing rarely used binaries and failed every native test. Tests and the testing helper now share a cache under XDG_CACHE_HOME or ~/.cache, and a damaged generation's error names the directory to remove. --- .../stack-catalog-setup.integration.test.ts | 4 +-- .../db/diff/diff.stack-cache.e2e.test.ts | 7 +++-- .../commands/db/reset/reset.stack.e2e.test.ts | 10 +++---- .../commands/db/start/start.roles.e2e.test.ts | 8 +++--- .../stack/restart/restart.integration.test.ts | 4 +-- .../stack/start/start.storage.e2e.test.ts | 5 ++-- .../functions/serve/serve.stack.e2e.test.ts | 8 +++--- apps/cli/tests/helpers/stack-artifacts.ts | 9 +++++++ .../stack/src/Commands.integration.test.ts | 4 +-- packages/stack/src/Owner.integration.test.ts | 4 +-- .../stack/src/StackHost.integration.test.ts | 27 ++++++++++--------- .../Supabase.native.integration.test.ts | 4 +-- .../stack/src/credentials.integration.test.ts | 6 ++--- .../stack/src/defaults.integration.test.ts | 4 +-- packages/stack/src/effect.integration.test.ts | 12 ++++----- .../stack/src/lifetime.integration.test.ts | 4 +-- .../stack/src/preparation/ArtifactStore.ts | 17 +++++++----- packages/stack/src/public.e2e.test.ts | 8 +++--- .../src/services/Database.integration.test.ts | 4 +-- .../services/Functions.integration.test.ts | 9 ++++--- .../src/services/Pooler.integration.test.ts | 3 ++- .../src/services/Rest.integration.test.ts | 6 ++--- .../StorageGateway.integration.test.ts | 4 +-- .../src/services/Vector.integration.test.ts | 4 +-- packages/stack/src/testing.ts | 9 ++++++- packages/stack/tests/artifact-cache.ts | 9 +++++++ packages/stack/tests/whole-stack/fixture.ts | 5 ++-- 27 files changed, 111 insertions(+), 87 deletions(-) create mode 100644 apps/cli/tests/helpers/stack-artifacts.ts create mode 100644 packages/stack/tests/artifact-cache.ts diff --git a/apps/cli/src/command-internal/stack-catalog-setup.integration.test.ts b/apps/cli/src/command-internal/stack-catalog-setup.integration.test.ts index 8601b9b809..a3e2e3d8c3 100644 --- a/apps/cli/src/command-internal/stack-catalog-setup.integration.test.ts +++ b/apps/cli/src/command-internal/stack-catalog-setup.integration.test.ts @@ -14,15 +14,15 @@ import { Stream, } from "effect"; import { FetchHttpClient } from "effect/unstable/http"; -import { tmpdir } from "node:os"; import { create, StackError, type Stack } from "@supabase/stack/effect"; import { postgres } from "@supabase/stack/commands"; import { mockOutput } from "../../tests/helpers/mocks.ts"; import type { Command } from "@supabase/stack/commands"; import { stackCatalogSetupLayer, StackCatalogSetup } from "./stack-catalog-setup.ts"; import { destroyTestStack } from "../../tests/helpers/stack-cleanup.ts"; +import { stackArtifactCacheRoot } from "../../tests/helpers/stack-artifacts.ts"; -const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; +const cacheRoot = stackArtifactCacheRoot; const jwtSecret = "stack-catalog-setup-integration-secret"; describe("stack catalog setup", { timeout: 180_000 }, () => { diff --git a/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts b/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts index 418d70771c..168035f975 100644 --- a/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts +++ b/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts @@ -3,9 +3,9 @@ import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Path, Redacted } from "effect"; import { FetchHttpClient } from "effect/unstable/http"; import { create as createStack, discover as discoverStacks } from "@supabase/stack/effect"; -import { tmpdir } from "node:os"; import { runSupabaseEffect } from "../../../../tests/helpers/cli.ts"; import { destroyTestStack } from "../../../../tests/helpers/stack-cleanup.ts"; +import { stackArtifactCacheRoot } from "../../../../tests/helpers/stack-artifacts.ts"; const COMMAND_TIMEOUT_MS = 8 * 60_000; const TEST_TIMEOUT_MS = COMMAND_TIMEOUT_MS * 4 + 2 * 60_000; @@ -91,10 +91,9 @@ describe("supabase db diff (stack shadow baseline cache)", () => { const home = yield* fs.makeTempDirectoryScoped({ prefix: `db-diff-cache-home-${runtime}-`, }); - const sharedArtifacts = path.join(tmpdir(), "supabase-stack-artifacts"); - yield* fs.makeDirectory(sharedArtifacts, { recursive: true }); + yield* fs.makeDirectory(stackArtifactCacheRoot, { recursive: true }); yield* fs.makeDirectory(path.join(home, "cache"), { recursive: true }); - yield* fs.symlink(sharedArtifacts, path.join(home, "cache", "stack")); + yield* fs.symlink(stackArtifactCacheRoot, path.join(home, "cache", "stack")); const supabase = path.join(project, "supabase"); const migrations = path.join(supabase, "migrations"); diff --git a/apps/cli/src/commands/db/reset/reset.stack.e2e.test.ts b/apps/cli/src/commands/db/reset/reset.stack.e2e.test.ts index f86b7e44f8..0c395b0ab1 100644 --- a/apps/cli/src/commands/db/reset/reset.stack.e2e.test.ts +++ b/apps/cli/src/commands/db/reset/reset.stack.e2e.test.ts @@ -4,10 +4,9 @@ import { Context, Data, Effect, FileSystem, Layer, Path, Redacted } from "effect import { FetchHttpClient, HttpClient } from "effect/unstable/http"; import { PgClient } from "@effect/sql-pg"; import { create as createStack } from "@supabase/stack/effect"; -import { tmpdir } from "node:os"; - import { runSupabaseEffect } from "../../../../tests/helpers/cli.ts"; import { destroyTestStack } from "../../../../tests/helpers/stack-cleanup.ts"; +import { stackArtifactCacheRoot } from "../../../../tests/helpers/stack-artifacts.ts"; const COMMAND_TIMEOUT_MS = 8 * 60_000; const TEST_TIMEOUT_MS = COMMAND_TIMEOUT_MS + 2 * 60_000; @@ -166,13 +165,10 @@ describe("supabase db reset (stack e2e)", () => { const home = yield* fs.makeTempDirectoryScoped({ prefix: `db-reset-home-${runtime}-` }); yield* writeFixture(root, fs, path); yield* fs.makeDirectory(path.join(home, "cache"), { recursive: true }); - yield* fs.makeDirectory(path.join(tmpdir(), "supabase-stack-artifacts"), { + yield* fs.makeDirectory(stackArtifactCacheRoot, { recursive: true, }); - yield* fs.symlink( - path.join(tmpdir(), "supabase-stack-artifacts"), - path.join(home, "cache", "stack"), - ); + yield* fs.symlink(stackArtifactCacheRoot, path.join(home, "cache", "stack")); const stack = yield* composeStack(root, home, runtime); diff --git a/apps/cli/src/commands/db/start/start.roles.e2e.test.ts b/apps/cli/src/commands/db/start/start.roles.e2e.test.ts index 51a83da8e8..11cd8ec67c 100644 --- a/apps/cli/src/commands/db/start/start.roles.e2e.test.ts +++ b/apps/cli/src/commands/db/start/start.roles.e2e.test.ts @@ -1,9 +1,8 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Path, Schema } from "effect"; -import { tmpdir } from "node:os"; - import { runSupabaseEffect } from "../../../../tests/helpers/cli.ts"; +import { stackArtifactCacheRoot } from "../../../../tests/helpers/stack-artifacts.ts"; const COMMAND_TIMEOUT_MS = 8 * 60_000; const AUX_TIMEOUT_MS = 3 * 60_000; @@ -27,9 +26,8 @@ describe("supabase db start (e2e, role alignment)", () => { const migrations = path.join(root, "supabase", "migrations"); yield* fs.makeDirectory(migrations, { recursive: true }); yield* fs.makeDirectory(path.join(home, "cache"), { recursive: true }); - const artifacts = path.join(tmpdir(), "supabase-stack-artifacts"); - yield* fs.makeDirectory(artifacts, { recursive: true }); - yield* fs.symlink(artifacts, path.join(home, "cache", "stack")); + yield* fs.makeDirectory(stackArtifactCacheRoot, { recursive: true }); + yield* fs.symlink(stackArtifactCacheRoot, path.join(home, "cache", "stack")); yield* fs.writeFileString( configPath, `project_id = "db-start-roles-e2e" diff --git a/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts b/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts index e848d47b84..f24df4b24d 100644 --- a/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts @@ -1,4 +1,3 @@ -import { tmpdir } from "node:os"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { StackError } from "@supabase/stack/effect"; @@ -11,6 +10,7 @@ import { import { StackApi, stackApiLayer, stackTargetResolverLayer } from "../stack.shared.ts"; import { stackRestart } from "./restart.handler.ts"; import { destroyTestStack } from "../../../../../tests/helpers/stack-cleanup.ts"; +import { stackArtifactCacheRoot } from "../../../../../tests/helpers/stack-artifacts.ts"; const live = Layer.provideMerge(stackApiLayer, BunServices.layer); const fixture = Effect.fn("StackRestartTest.fixture")(function* () { @@ -20,7 +20,7 @@ const fixture = Effect.fn("StackRestartTest.fixture")(function* () { const api = yield* StackApi; const locations = { stateRoot: path.join(root, "stacks"), - cacheRoot: path.join(tmpdir(), "supabase-stack-artifacts"), + cacheRoot: stackArtifactCacheRoot, }; const stack = yield* api.create({ ...locations, projectRoot: root, runtime: "native" }); const output = mockOutput(); diff --git a/apps/cli/src/commands/experimental/stack/start/start.storage.e2e.test.ts b/apps/cli/src/commands/experimental/stack/start/start.storage.e2e.test.ts index c268717965..5c6c6431ca 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.storage.e2e.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.storage.e2e.test.ts @@ -2,13 +2,12 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { Data, Effect, Exit, FileSystem, Path, Schema } from "effect"; import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http"; -import { tmpdir } from "node:os"; - import { makeTempCliProject, makeTempHome, runSupabaseEffect, } from "../../../../../tests/helpers/cli.ts"; +import { stackArtifactCacheRoot } from "../../../../../tests/helpers/stack-artifacts.ts"; const COMMAND_TIMEOUT_MS = 10 * 60_000; const CLEANUP_TIMEOUT_MS = 120_000; @@ -92,7 +91,7 @@ describe("stack start Storage behind the API gateway (compiled e2e)", () => { const project = yield* Effect.promise(() => makeTempCliProject(`stack-storage-${runtime}-e2e-`), ); - const artifacts = path.join(tmpdir(), "supabase-stack-artifacts"); + const artifacts = stackArtifactCacheRoot; yield* fs.makeDirectory(path.join(home.dir, "cache"), { recursive: true }); yield* fs.makeDirectory(artifacts, { recursive: true }); yield* fs.symlink(artifacts, path.join(home.dir, "cache", "stack")); diff --git a/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts b/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts index a860a48d3e..07cdf21230 100644 --- a/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts +++ b/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts @@ -14,11 +14,12 @@ import { Stream, } from "effect"; import { FetchHttpClient, HttpClient } from "effect/unstable/http"; -import { homedir, tmpdir } from "node:os"; +import { homedir } from "node:os"; import { spawnSupabase } from "../../../../tests/helpers/cli.ts"; import { generateGoJwt } from "../../../command-internal/go-jwt.ts"; import { destroyTestStack } from "../../../../tests/helpers/stack-cleanup.ts"; +import { stackArtifactCacheRoot } from "../../../../tests/helpers/stack-artifacts.ts"; const jwtSecret = "functions-serve-stack-e2e-secret-at-least-32-characters"; const nativeSupported = @@ -66,9 +67,8 @@ const fixture = Effect.fn("FunctionsServeE2e.fixture")(function* ( ); yield* fs.writeFileString(path.join(root, "override.env"), "CUSTOM_VALUE=overridden\n"); yield* fs.makeDirectory(path.join(home, "cache"), { recursive: true }); - const artifacts = path.join(tmpdir(), "supabase-stack-artifacts"); - yield* fs.makeDirectory(artifacts, { recursive: true }); - yield* fs.symlink(artifacts, path.join(home, "cache", "stack")); + yield* fs.makeDirectory(stackArtifactCacheRoot, { recursive: true }); + yield* fs.symlink(stackArtifactCacheRoot, path.join(home, "cache", "stack")); const stack = yield* createStack({ projectRoot: root, stateRoot: path.join(home, "stacks"), diff --git a/apps/cli/tests/helpers/stack-artifacts.ts b/apps/cli/tests/helpers/stack-artifacts.ts new file mode 100644 index 0000000000..70e16863aa --- /dev/null +++ b/apps/cli/tests/helpers/stack-artifacts.ts @@ -0,0 +1,9 @@ +import { homedir } from "node:os"; // oxlint-disable-line effecttsgo/node-builtin-import -- plain constant shared by test files. +import { join } from "node:path"; // oxlint-disable-line effecttsgo/node-builtin-import -- plain constant shared by test files. + +/** Shared native artifact cache for stack tests; OS temp directories age out files inside published generations. */ +export const stackArtifactCacheRoot = join( + // oxlint-disable-next-line effecttsgo/process-env -- plain constant shared by test files. + process.env.XDG_CACHE_HOME || join(homedir(), ".cache"), + "supabase-stack-artifacts", +); diff --git a/packages/stack/src/Commands.integration.test.ts b/packages/stack/src/Commands.integration.test.ts index c27b95b6f0..ef44dfb3ac 100644 --- a/packages/stack/src/Commands.integration.test.ts +++ b/packages/stack/src/Commands.integration.test.ts @@ -15,7 +15,6 @@ import { Schedule, Stream, } from "effect"; -import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; import { postgres } from "./Commands.ts"; import type { ContainerEngine } from "./runtime/Container.ts"; @@ -27,8 +26,9 @@ import { makeStandaloneService } from "../tests/standalone-service.ts"; import { bindTcp, serveTcp } from "./Proxy.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; import { engineTarget, testEngine } from "../tests/engine-target.ts"; +import { testArtifactCacheRoot } from "../tests/artifact-cache.ts"; -const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; +const cacheRoot = testArtifactCacheRoot; class PgProveTestError extends Data.TaggedError("PgProveTestError")<{ readonly message: string; diff --git a/packages/stack/src/Owner.integration.test.ts b/packages/stack/src/Owner.integration.test.ts index 7f5fcff9df..5ab9c70b54 100644 --- a/packages/stack/src/Owner.integration.test.ts +++ b/packages/stack/src/Owner.integration.test.ts @@ -5,7 +5,6 @@ import { Context, Effect, FileSystem, Layer, Redacted, Ref, Schema, Stream } fro import { HttpClient } from "effect/unstable/http"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { randomUUID } from "node:crypto"; -import { tmpdir } from "node:os"; import { RpcTest } from "effect/unstable/rpc"; import * as Owner from "./Owner.ts"; import { OwnerRpc } from "./Rpc.ts"; @@ -17,6 +16,7 @@ import type { ServiceCreationInput } from "./services/Catalog.ts"; import { ownerFor } from "../tests/owner-rpc.ts"; import { engineTarget, testEngine } from "../tests/engine-target.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; +import { testArtifactCacheRoot } from "../tests/artifact-cache.ts"; const stateFor = (root: string) => Effect.gen(function* () { @@ -24,7 +24,7 @@ const stateFor = (root: string) => return Context.get(context, StackNamespace.Service); }); -const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; +const cacheRoot = testArtifactCacheRoot; const initial = (id: string): SavedStack => ({ id, diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts index b3d31405dc..bfb235ec98 100644 --- a/packages/stack/src/StackHost.integration.test.ts +++ b/packages/stack/src/StackHost.integration.test.ts @@ -36,6 +36,7 @@ import { bindControl, makeRuntime } from "./StackHost.ts"; import { shutdownOwner } from "../tests/owner.ts"; import { postgres } from "./Commands.ts"; import * as CommandRunner from "./host/CommandRunner.ts"; +import { testArtifactCacheRoot } from "../tests/artifact-cache.ts"; class HostTestError extends Data.TaggedError("HostTestError")<{ readonly message: string }> {} @@ -147,7 +148,7 @@ const inProcessRuntime = ( CommandRunner.layer({ stackId: "stack", root, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, runtime: "native", }), ); @@ -235,7 +236,7 @@ it.live("preserves composition outcomes over RPC", () => saved, state, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const { runtime } = yield* inProcessRuntime(owner, state, root); const client = yield* ownerClient(runtime.access); @@ -290,7 +291,7 @@ it.live("stops serving after a failed shutdown and keeps reporting that failure" saved, state, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const failedOwner = { ...owner, @@ -339,7 +340,7 @@ it.live( saved, state, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const stopGate = yield* Deferred.make(); const destroyEntered = yield* Deferred.make(); @@ -397,7 +398,7 @@ it.live("reports a destroy failure and the owner exits", () => saved, state, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const failedOwner = { ...owner, @@ -460,7 +461,7 @@ it.live("rejects destroy while stop is in flight", () => saved, state, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const entered = yield* Deferred.make(); const release = yield* Deferred.make(); @@ -505,7 +506,7 @@ it.live("exits after a namespace shutdown defect without rerunning cleanup", () saved, state, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const failStop = yield* Ref.make(true); const failedOwner = { @@ -547,7 +548,7 @@ it.live( }); const access = yield* launchHost(state, { stateRoot: `${root}/state`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, stackId: "stack", }); const { endpoint } = access; @@ -754,7 +755,7 @@ it.live("finishes detached shutdown after the caller disconnects", () => }, state, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const entered = yield* Deferred.make(); const allow = yield* Deferred.make(); @@ -798,7 +799,7 @@ it.live("withdraws a command waiting for its prerequisite", () => saved, state, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const entered = yield* Deferred.make(); const cancelled = yield* Deferred.make(); @@ -880,7 +881,7 @@ it.live("finishes a service creation after its caller disconnects", () => ), }, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const interrupted = yield* Deferred.make(); const { runtime } = yield* inProcessRuntime( @@ -942,7 +943,7 @@ it.live("persists a composition change after its caller disconnects", () => ).pipe(Effect.andThen(state.save(next))), }, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const interrupted = yield* Deferred.make(); const { runtime } = yield* inProcessRuntime( @@ -1008,7 +1009,7 @@ const abandonedComposition = (prefix: string, destroy: boolean) => ), }, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const interrupted = yield* Deferred.make(); const { runtime } = yield* inProcessRuntime( diff --git a/packages/stack/src/composition/Supabase.native.integration.test.ts b/packages/stack/src/composition/Supabase.native.integration.test.ts index 21f42b67c7..d99a7a695e 100644 --- a/packages/stack/src/composition/Supabase.native.integration.test.ts +++ b/packages/stack/src/composition/Supabase.native.integration.test.ts @@ -15,7 +15,6 @@ import { import { PgClient } from "@effect/sql-pg"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { SignJWT } from "jose"; -import { tmpdir } from "node:os"; import * as StackNamespace from "../StackNamespace.ts"; import type { SavedStack } from "../StackNamespace.ts"; import { ownerFor } from "../../tests/owner-rpc.ts"; @@ -24,8 +23,9 @@ import { SupabaseCompositionError, type SupabaseCompositionOperations, } from "./Supabase.ts"; +import { testArtifactCacheRoot } from "../../tests/artifact-cache.ts"; -const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; +const cacheRoot = testArtifactCacheRoot; // Below every OS ephemeral range, so another test's outbound socket cannot already hold them. const FIXED_STUDIO_PORT = 24_391; const FIXED_MAIL_PORT = 24_392; diff --git a/packages/stack/src/credentials.integration.test.ts b/packages/stack/src/credentials.integration.test.ts index c66ce6dfc6..2f704f035a 100644 --- a/packages/stack/src/credentials.integration.test.ts +++ b/packages/stack/src/credentials.integration.test.ts @@ -1,8 +1,8 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; import { Effect, FileSystem, Layer, Redacted } from "effect"; -import { tmpdir } from "node:os"; import { create, open } from "./effect.ts"; +import { testArtifactCacheRoot } from "../tests/artifact-cache.ts"; const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); @@ -14,7 +14,7 @@ it.live("persists effective credentials across service creation, reopen, and res const options = { projectRoot: root, stateRoot: `${root}/state`, - cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + cacheRoot: testArtifactCacheRoot, runtime: "native", } as const; const stack = yield* create(options); @@ -53,7 +53,7 @@ it.live("resolves composition credentials before creating services", () => const stack = yield* create({ projectRoot: root, stateRoot: `${root}/state`, - cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + cacheRoot: testArtifactCacheRoot, runtime: "native", }); yield* Effect.addFinalizer(() => stack.destroy.pipe(Effect.orDie)); diff --git a/packages/stack/src/defaults.integration.test.ts b/packages/stack/src/defaults.integration.test.ts index af71143ea4..1f0b3ab928 100644 --- a/packages/stack/src/defaults.integration.test.ts +++ b/packages/stack/src/defaults.integration.test.ts @@ -1,12 +1,12 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Layer, Redacted } from "effect"; -import { tmpdir } from "node:os"; import { DEFAULT_LOCAL_JWT_SECRET, DEFAULT_POSTGRES_ROOT_KEY } from "./Defaults.ts"; import { create as createEffect, StackError } from "./effect.ts"; import { create as createPromise } from "./index.ts"; +import { testArtifactCacheRoot } from "../tests/artifact-cache.ts"; -const artifactCacheRoot = `${tmpdir()}/supabase-stack-artifacts`; +const artifactCacheRoot = testArtifactCacheRoot; const effectLayer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); const effectDatabaseConfig = { diff --git a/packages/stack/src/effect.integration.test.ts b/packages/stack/src/effect.integration.test.ts index 34423e1401..0fdfdcaf37 100644 --- a/packages/stack/src/effect.integration.test.ts +++ b/packages/stack/src/effect.integration.test.ts @@ -14,7 +14,6 @@ import { } from "effect"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- the test binds a dead owner's exact port. import * as Net from "node:net"; -import { tmpdir } from "node:os"; import { create, discover, @@ -35,6 +34,7 @@ import { foreignRelease } from "../tests/release-owner-fixture.ts"; import { destroyTestStack } from "../tests/stack-cleanup.ts"; import { watchEntry } from "../tests/watch-entry.ts"; import { deriveStackId, resolveStackIdentity } from "./identity/Identity.ts"; +import { testArtifactCacheRoot } from "../tests/artifact-cache.ts"; const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); // Below every OS ephemeral range, so another test's outbound socket cannot already hold it. @@ -225,7 +225,7 @@ it.live( const options = { projectRoot: root, stateRoot: `${root}/state`, - cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + cacheRoot: testArtifactCacheRoot, runtime: "native", } satisfies Parameters[0]; const stack = yield* create(options); @@ -273,7 +273,7 @@ const resetDataStory = (runtime: "native" | ContainerEngine) => const options = { projectRoot: root, stateRoot: `${root}/state`, - cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + cacheRoot: testArtifactCacheRoot, runtime, } satisfies Parameters[0]; const stack = yield* create(options); @@ -749,7 +749,7 @@ it.live( const options = { projectRoot: root, stateRoot: `${root}/state`, - cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + cacheRoot: testArtifactCacheRoot, runtime: "native", } satisfies Parameters[0]; const handleScope = yield* Scope.make(); @@ -866,7 +866,7 @@ it.live( const options = { projectRoot: root, stateRoot: `${root}/state`, - cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + cacheRoot: testArtifactCacheRoot, runtime: "native", } satisfies Parameters[0]; const stack = yield* create(options); @@ -1164,7 +1164,7 @@ it.live( const options = { projectRoot: root, stateRoot: `${root}/state`, - cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + cacheRoot: testArtifactCacheRoot, runtime: "native", } satisfies Parameters[0]; const secret = "stop-order-secret-with-at-least-thirty-two-characters"; diff --git a/packages/stack/src/lifetime.integration.test.ts b/packages/stack/src/lifetime.integration.test.ts index 7710523eb0..654d26e68a 100644 --- a/packages/stack/src/lifetime.integration.test.ts +++ b/packages/stack/src/lifetime.integration.test.ts @@ -2,7 +2,6 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; import { Context, Data, Effect, Fiber, FileSystem, Layer, Option, Schema, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; import { create, open } from "./effect.ts"; import { launchHost, ownerClient, ownerExitProbe, waitForOwnerExit } from "./HostProcess.ts"; @@ -14,13 +13,14 @@ import { watchLeaseRelease, } from "../tests/owner.ts"; import { watchEntry } from "../tests/watch-entry.ts"; +import { testArtifactCacheRoot } from "../tests/artifact-cache.ts"; class LifetimeTestError extends Data.TaggedError("LifetimeTestError")<{ readonly message: string; }> {} const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); -const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; +const cacheRoot = testArtifactCacheRoot; const sessionFixture = fileURLToPath( new URL("../tests/session-client-fixture.ts", import.meta.url), ); diff --git a/packages/stack/src/preparation/ArtifactStore.ts b/packages/stack/src/preparation/ArtifactStore.ts index eb57182ef0..68719d266c 100644 --- a/packages/stack/src/preparation/ArtifactStore.ts +++ b/packages/stack/src/preparation/ArtifactStore.ts @@ -422,9 +422,10 @@ const inspectRequiredPaths = ( ), ); if (!exists) - return yield* metadataError("Cached artifact is missing a required runtime path", { - path: relative, - }); + return yield* metadataError( + `Cached artifact is missing ${candidate}; remove ${root} to reinstall it`, + { path: relative }, + ); const inspected = yield* inspectFreshPath(fs, path, candidate, realRoot); entries.push([relative, inspected]); } @@ -458,6 +459,7 @@ const ensureExecutableFile = ( /** A published generation is never chmodded again, so a cache hit must check the bit directly. */ const ensureExecutableMode = ( fs: FileSystem.FileSystem, + generation: string, executable: string, ): Effect.Effect => fs.stat(executable).pipe( @@ -471,9 +473,10 @@ const ensureExecutableMode = ( (info.mode & 0o100) !== 0 ? Effect.void : Effect.fail( - metadataError("Cached artifact executable is missing its executable bit", { - path: executable, - }), + metadataError( + `Cached artifact executable ${executable} is missing its executable bit; remove ${generation} to reinstall it`, + { path: executable }, + ), ), ), ); @@ -735,7 +738,7 @@ const checkHit = Effect.fn("ArtifactStore.checkHit")(function* ( path: request.executablePath, }); yield* ensureExecutableFile(fs, executable.realPath); - yield* ensureExecutableMode(fs, executable.realPath); + yield* ensureExecutableMode(fs, resolved.generationPath, executable.realPath); } return Option.some({ key: request.key, diff --git a/packages/stack/src/public.e2e.test.ts b/packages/stack/src/public.e2e.test.ts index f8f33a415c..c27754776b 100644 --- a/packages/stack/src/public.e2e.test.ts +++ b/packages/stack/src/public.e2e.test.ts @@ -3,12 +3,12 @@ import { expect, expectTypeOf, it } from "@effect/vitest"; import { Effect, FileSystem, Layer, Path, Redacted, Schema, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { HttpClient } from "effect/unstable/http"; -import { tmpdir } from "node:os"; import { discover, open } from "./effect.ts"; import { postgres } from "./Commands.ts"; import * as PromiseStack from "./index.ts"; import { assertOwnerExited, captureOwnerPid } from "../tests/owner.ts"; import { destroyTestStack } from "../tests/stack-cleanup.ts"; +import { testArtifactCacheRoot } from "../tests/artifact-cache.ts"; for (const runtime of ["node", "bun"] as const) { it.live( @@ -22,7 +22,7 @@ for (const runtime of ["node", "bun"] as const) { [ new URL("../tests/owner-exit-client.ts", import.meta.url).pathname, root, - `${tmpdir()}/supabase-stack-artifacts`, + testArtifactCacheRoot, ], { stdin: "ignore", stdout: "pipe", stderr: "pipe" }, ); @@ -51,7 +51,7 @@ it.live( const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-public-e2e-" }); - const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; + const cacheRoot = testArtifactCacheRoot; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( ChildProcess.make( @@ -370,7 +370,7 @@ it.live( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-public-observation-" }); - const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; + const cacheRoot = testArtifactCacheRoot; const locations = { stateRoot: `${root}/state`, cacheRoot }; const stack = yield* Effect.tryPromise(() => PromiseStack.create({ ...locations, projectRoot: root, runtime: "native" }), diff --git a/packages/stack/src/services/Database.integration.test.ts b/packages/stack/src/services/Database.integration.test.ts index cc1117614d..8b00cdef9d 100644 --- a/packages/stack/src/services/Database.integration.test.ts +++ b/packages/stack/src/services/Database.integration.test.ts @@ -15,13 +15,13 @@ import { Stream, } from "effect"; import { createHash } from "node:crypto"; -import { tmpdir } from "node:os"; import { DEFAULT_POSTGRES_ROOT_KEY } from "../Defaults.ts"; import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeDatabase, type BackendEndpoint, type DatabaseConfig } from "./Database.ts"; import { makeDockerDatabaseRoot, runEngine } from "../../tests/docker-fixture.ts"; import { observeContainerStop, cleanStopEvents } from "../../tests/engine-events.ts"; import { engineTarget, testEngine } from "../../tests/engine-target.ts"; +import { testArtifactCacheRoot } from "../../tests/artifact-cache.ts"; const config: DatabaseConfig = { version: "17", @@ -31,7 +31,7 @@ const config: DatabaseConfig = { rootKey: Redacted.make("a".repeat(64)), }; -const artifactCacheRoot = `${tmpdir()}/supabase-stack-artifacts`; +const artifactCacheRoot = testArtifactCacheRoot; const query = ( endpoint: BackendEndpoint, diff --git a/packages/stack/src/services/Functions.integration.test.ts b/packages/stack/src/services/Functions.integration.test.ts index a91b59c1a2..17b953b3e6 100644 --- a/packages/stack/src/services/Functions.integration.test.ts +++ b/packages/stack/src/services/Functions.integration.test.ts @@ -24,6 +24,7 @@ import * as Functions from "./Functions.ts"; import { makeProcessRecipe } from "./ProcessRecipe.ts"; import { engineTarget, testEngine } from "../../tests/engine-target.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; +import { testArtifactCacheRoot } from "../../tests/artifact-cache.ts"; const options = (root: string) => ({ stackId: "catalog-functions", @@ -260,7 +261,7 @@ describe("service catalog", () => { ...options(stackRoot), stackId: "d".repeat(64), instanceId: "ancestor", - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }, ); const logs = yield* Ref.make(""); @@ -333,7 +334,7 @@ describe("service catalog", () => { ...options(stackRoot), stackId: "e".repeat(64), instanceId: "deno-config", - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }, ); const logs = yield* Ref.make(""); @@ -407,7 +408,7 @@ describe("service catalog", () => { ...options(stackRoot), stackId: "f".repeat(64), instanceId: "plain-deno-config", - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }, ); const logs = yield* Ref.make(""); @@ -522,7 +523,7 @@ for (const runtime of ["native", testEngine] as const) { instanceId: "configured", runtime, ...(runtime !== "native" ? { engineTarget } : {}), - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }, ); const logs = yield* Ref.make(""); diff --git a/packages/stack/src/services/Pooler.integration.test.ts b/packages/stack/src/services/Pooler.integration.test.ts index f35a295426..9cb5ebbc88 100644 --- a/packages/stack/src/services/Pooler.integration.test.ts +++ b/packages/stack/src/services/Pooler.integration.test.ts @@ -8,12 +8,13 @@ import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { engineTarget, testEngine } from "../../tests/engine-target.ts"; +import { testArtifactCacheRoot } from "../../tests/artifact-cache.ts"; const options = (root: string) => ({ stackId: "catalog-pooler", instanceId: "instance", root, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, runtime: "native" as const, }); diff --git a/packages/stack/src/services/Rest.integration.test.ts b/packages/stack/src/services/Rest.integration.test.ts index 55e8a56867..1dbcdb52b8 100644 --- a/packages/stack/src/services/Rest.integration.test.ts +++ b/packages/stack/src/services/Rest.integration.test.ts @@ -4,7 +4,6 @@ import { Context, Effect, FileSystem, Layer, Path, Redacted, Ref } from "effect" import { PgClient } from "@effect/sql-pg"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { SignJWT } from "jose"; -import { tmpdir } from "node:os"; import * as Network from "../Network.ts"; import * as StackNamespace from "../StackNamespace.ts"; import { makeStandaloneService } from "../../tests/standalone-service.ts"; @@ -13,6 +12,7 @@ import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { engineTarget, testEngine } from "../../tests/engine-target.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; +import { testArtifactCacheRoot } from "../../tests/artifact-cache.ts"; const makeTestState = (root: string) => Layer.build(StackNamespace.layer({ root })).pipe( @@ -218,7 +218,7 @@ describe("service catalog", () => { jwtExpiry: 3600, }, }, - { ...options(root), stackId, cacheRoot: `${tmpdir()}/supabase-stack-artifacts` }, + { ...options(root), stackId, cacheRoot: testArtifactCacheRoot }, ); const database = yield* makeStandaloneService(databaseRecipe.definition, { id: "database", @@ -261,7 +261,7 @@ describe("service catalog", () => { anonRole: "anon", }, }, - { ...options(root), stackId, cacheRoot: `${tmpdir()}/supabase-stack-artifacts` }, + { ...options(root), stackId, cacheRoot: testArtifactCacheRoot }, ); const rest = yield* makeStandaloneService(restRecipe.definition, { id: "rest", diff --git a/packages/stack/src/services/StorageGateway.integration.test.ts b/packages/stack/src/services/StorageGateway.integration.test.ts index e41a0af9d2..e5fef13ec2 100644 --- a/packages/stack/src/services/StorageGateway.integration.test.ts +++ b/packages/stack/src/services/StorageGateway.integration.test.ts @@ -2,7 +2,6 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; import { Context, Crypto, Effect, FileSystem, Layer, Option, Path, Redacted } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; -import { tmpdir } from "node:os"; import { DEFAULT_LOCAL_S3_ACCESS_KEY_ID, DEFAULT_LOCAL_S3_REGION, @@ -13,8 +12,9 @@ import type { SavedStack } from "../StackNamespace.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { ownerFor } from "../../tests/owner-rpc.ts"; import { engineTarget, testEngine } from "../../tests/engine-target.ts"; +import { testArtifactCacheRoot } from "../../tests/artifact-cache.ts"; -const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; +const cacheRoot = testArtifactCacheRoot; const jwtSecret = "storage-gateway-secret-with-at-least-32-chars"; const s3Credentials = { accessKeyId: DEFAULT_LOCAL_S3_ACCESS_KEY_ID, diff --git a/packages/stack/src/services/Vector.integration.test.ts b/packages/stack/src/services/Vector.integration.test.ts index 3fca719cd7..1a8df9ebc9 100644 --- a/packages/stack/src/services/Vector.integration.test.ts +++ b/packages/stack/src/services/Vector.integration.test.ts @@ -1,6 +1,5 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; -import { tmpdir } from "node:os"; import { Effect, FileSystem, Layer } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { makeStandaloneService } from "../../tests/standalone-service.ts"; @@ -8,12 +7,13 @@ import { makeServiceRecipe } from "./Catalog.ts"; import type { ContainerEngine } from "../runtime/Container.ts"; import { engineTarget, testEngine } from "../../tests/engine-target.ts"; import { httpHost } from "../../tests/helpers/endpoint.ts"; +import { testArtifactCacheRoot } from "../../tests/artifact-cache.ts"; const options = (root: string, runtime: ContainerEngine | "native") => ({ stackId: "catalog-test", instanceId: "vector", root, - cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + cacheRoot: testArtifactCacheRoot, runtime, ...(runtime !== "native" ? { engineTarget } : {}), }); diff --git a/packages/stack/src/testing.ts b/packages/stack/src/testing.ts index b92c16d860..a34efc92db 100644 --- a/packages/stack/src/testing.ts +++ b/packages/stack/src/testing.ts @@ -117,6 +117,8 @@ const runtimeOverride = Config.option( Config.literals(["native", "docker", "podman"], "SUPABASE_STACK_TEST_RUNTIME"), ); +const cacheHome = Config.option(Config.nonEmptyString("XDG_CACHE_HOME")); + const testFailure = (operation: string) => (cause: unknown) => stackError(operation, cause); /** Local-development configuration for each kind, with every endpoint on an automatic port. */ @@ -167,7 +169,12 @@ const make = Effect.fn("TestStack.make")( // Shared roots are created owner-only, so each OS user needs its own. const user = process.getuid?.() ?? userInfo().username; const stateRoot = options.stateRoot ?? path.join(tmpdir(), `supabase-stack-tests-${user}`); - const cacheRoot = options.cacheRoot ?? path.join(tmpdir(), `supabase-stack-artifacts-${user}`); + const cacheRoot = + options.cacheRoot ?? + path.join( + Option.getOrElse(yield* cacheHome, () => path.join(homedir(), ".cache")), + "supabase-stack-artifacts", + ); // Native Edge Runtime on Linux cannot read project files below /tmp. const temporaryRoot = fs.makeTempDirectoryScoped({ prefix: "supabase-test-stack-", diff --git a/packages/stack/tests/artifact-cache.ts b/packages/stack/tests/artifact-cache.ts new file mode 100644 index 0000000000..9853e7ae48 --- /dev/null +++ b/packages/stack/tests/artifact-cache.ts @@ -0,0 +1,9 @@ +import { homedir } from "node:os"; // oxlint-disable-line effecttsgo/node-builtin-import -- plain constant shared by test files. +import { join } from "node:path"; // oxlint-disable-line effecttsgo/node-builtin-import -- plain constant shared by test files. + +/** Shared native artifact cache for tests; OS temp directories age out files inside published generations. */ +export const testArtifactCacheRoot = join( + // oxlint-disable-next-line effecttsgo/process-env -- plain constant shared by test files. + process.env.XDG_CACHE_HOME || join(homedir(), ".cache"), + "supabase-stack-artifacts", +); diff --git a/packages/stack/tests/whole-stack/fixture.ts b/packages/stack/tests/whole-stack/fixture.ts index cf7c258e9c..0cd6f56fe9 100644 --- a/packages/stack/tests/whole-stack/fixture.ts +++ b/packages/stack/tests/whole-stack/fixture.ts @@ -13,7 +13,7 @@ import { Stream, } from "effect"; import { postgres } from "../../src/Commands.ts"; -import { homedir, tmpdir } from "node:os"; +import { homedir } from "node:os"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { create, type Stack } from "../../src/effect.ts"; import type { ContainerEngine } from "../../src/runtime/Container.ts"; @@ -22,6 +22,7 @@ import { vectorAnalyticsConfig } from "./analytics.ts"; import { cleanupDockerRoot } from "../docker-cleanup.ts"; import { destroyTestStack } from "../stack-cleanup.ts"; import { testEngine } from "../test-engine.ts"; +import { testArtifactCacheRoot } from "../artifact-cache.ts"; type AnyService = Effect.Success[number]; @@ -138,7 +139,7 @@ export const wholeStack = Effect.fn("WholeStack.fixture")((runtime: Runtime) => const secret = `whole-stack-${yield* crypto.randomUUIDv4}-secret`; const locations = { stateRoot: `${root}/state`, - cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + cacheRoot: testArtifactCacheRoot, }; const stack = yield* create({ projectRoot: root, From 3f887315296a45bacaea4250cf856a6a4324a00b Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 09:21:34 +0200 Subject: [PATCH 117/142] fix(stack): refresh stopped credential consumers when stack credentials change A credential change saved the new record but left every stopped standalone consumer with the previous values baked into its creation, so its next start signed and verified with stale keys. The change now refreshes those creations in the same locked save, keeping values set explicitly by the caller. --- packages/stack/src/Owner.integration.test.ts | 55 +++++++++++++++++++- packages/stack/src/Owner.ts | 33 +++++++++++- packages/stack/src/host/Credentials.ts | 33 +++++++++--- 3 files changed, 112 insertions(+), 9 deletions(-) diff --git a/packages/stack/src/Owner.integration.test.ts b/packages/stack/src/Owner.integration.test.ts index 5ab9c70b54..3bfeef2ef1 100644 --- a/packages/stack/src/Owner.integration.test.ts +++ b/packages/stack/src/Owner.integration.test.ts @@ -350,7 +350,7 @@ it.live("forwards and rotates saved identity across composed services in one own const excludedStudio = (yield* owner.rpc.status({ id: studioId })).config; expect(excludedStudio.service).toBe("studio"); if (excludedStudio.service === "studio") - expect(excludedStudio.config.anonKey).toBe("anon-one"); + expect(excludedStudio.config.anonKey).toBe(secondCredentials.anonKey); for (const id of retainedIds) expect((yield* owner.rpc.status({ id: id })).lifecycle).toBe("stopped"); @@ -973,6 +973,59 @@ it.live("blocks a credential change while a standalone credential consumer runs" ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); +it.live( + "refreshes a stopped standalone credential consumer when the stack credentials change", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-owner-credential-refresh-", + }); + const stack = initial("owner-credential-refresh"); + const state = yield* stateFor(`${root}/state`); + yield* state.save(stack); + const owner = yield* ownerFor({ saved: stack, state, root: `${root}/data`, cacheRoot }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + const rest = yield* owner.rpc.createService({ + service: "rest", + config: { databaseUrl: "postgresql://authenticator@127.0.0.1:1/postgres" }, + endpoints: {}, + }); + const explicitKeys = '[{"kty":"oct","k":"explicit"}]'; + const auth = yield* owner.rpc.createService({ + service: "auth", + config: { gotrueJwtKeys: explicitKeys }, + endpoints: {}, + }); + const before = yield* owner.getStackCredentials; + + yield* owner.rpc.supabaseComposition({ + services: [{ service: "mail", config: {}, endpoints: {} }], + keys: { + gotrueJwtKeys: "[]", + publicSigningKeys: '[{"kty":"EC","kid":"rotated"}]', + }, + }); + + const after = yield* owner.getStackCredentials; + expect(after.jwks).not.toBe(before.jwks); + const observed = yield* owner.rpc.status({ id: rest.id }); + expect(observed.config).toMatchObject({ config: { jwks: after.jwks } }); + const saved = yield* state.read(stack.id); + expect(saved?.instances.find(({ id }) => id === rest.id)?.creation).toMatchObject({ + config: { jwks: after.jwks }, + }); + expect((yield* owner.rpc.status({ id: auth.id })).config).toMatchObject({ + config: { gotrueJwtKeys: explicitKeys }, + }); + + yield* owner.rpc.startService({ id: rest.id }); + yield* owner.rpc.stopService({ id: rest.id }); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + it.live("rejects a missing required input before starting or stopping the service", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index dd4752d50e..4d6c5420af 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -43,6 +43,7 @@ import { CredentialError, credentialOverrides, nextCredentials, + refreshCredentials, withCredentials, withoutUnusedCredentials, } from "./host/Credentials.ts"; @@ -342,8 +343,36 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { const current = yield* readSaved; const next = yield* nextCredentials(current, overrides, keys); if (next === current.credentials) return next; - if (current.credentials !== undefined) yield* requireStopped(current); - yield* options.state.save({ ...current, credentials: next }); + const previous = current.credentials; + if (previous === undefined) { + yield* options.state.save({ ...current, credentials: next }); + return next; + } + yield* requireStopped(current); + const refreshed = new Map( + yield* Effect.forEach( + current.instances.filter(({ creation }) => consumesCredentials(creation)), + ({ id, creation }) => + refreshCredentials(creation, previous, next).pipe( + Effect.map((refreshedCreation) => [id, refreshedCreation] as const), + ), + ), + ); + yield* options.state.save({ + ...current, + credentials: next, + instances: current.instances.map((instance) => { + const creation = refreshed.get(instance.id); + return creation === undefined ? instance : { ...instance, creation }; + }), + }); + yield* Effect.forEach( + refreshed, + ([id, creation]) => + orchestrator.get(id).pipe(Effect.flatMap((entry) => Ref.set(entry.creation, creation))), + { discard: true }, + ); + yield* Effect.annotateCurrentSpan("refreshed_instances", refreshed.size); return next; }), ); diff --git a/packages/stack/src/host/Credentials.ts b/packages/stack/src/host/Credentials.ts index 65e5017e33..152f9c43a9 100644 --- a/packages/stack/src/host/Credentials.ts +++ b/packages/stack/src/host/Credentials.ts @@ -191,15 +191,11 @@ const credentialInputs: { export const credentialInputNames = (service: ServiceCreation["service"]): ReadonlyArray => Object.keys(credentialInputs[service]); -/** - * Completes a creation with the stack credentials it consumes. The JWT secret and database - * credentials always come from the record; other inputs keep an explicit value. - */ -export const withCredentials = ( +const fillCredentials = ( creation: ServiceCreationInput, + config: Record, credentials: StackCredentials, ): Effect.Effect => { - const config: Record = { ...creation.config }; for (const [input, source] of Object.entries(credentialInputs[creation.service])) { const value = credentials[source]; if (creation.service === "database") config[input] = Redacted.make(value); @@ -207,3 +203,28 @@ export const withCredentials = ( } return Schema.decodeUnknownEffect(ServiceCreation)({ ...creation, config }); }; + +/** + * Completes a creation with the stack credentials it consumes. The JWT secret and database + * credentials always come from the record; other inputs keep an explicit value. + */ +export const withCredentials = ( + creation: ServiceCreationInput, + credentials: StackCredentials, +): Effect.Effect => + fillCredentials(creation, { ...creation.config }, credentials); + +/** + * Moves a saved creation from the previous credential record to the next. An input that differs + * from the previous record is explicit and kept; one equal to it follows the next record. + */ +export const refreshCredentials = ( + creation: ServiceCreation, + previous: StackCredentials, + next: StackCredentials, +): Effect.Effect => { + const config: Record = { ...creation.config }; + for (const [input, source] of Object.entries(credentialInputs[creation.service])) + if (config[input] === previous[source]) delete config[input]; + return fillCredentials(creation, config, next); +}; From e5f0f61051375aa1408d5ca70e14658fb7ccab55 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 10:14:09 +0200 Subject: [PATCH 118/142] test(cli): prove a service was not relaunched through its runtime identity A follower that is still catching up could miss the statuses of an unintended relaunch, and a stop barrier could be produced by that relaunch itself. Tests now compare per-process state captured before and after: Mailpit's accepted-message count and the Postgres start time. --- .../stack/restart/restart.integration.test.ts | 55 ++++++++++---- .../functions/serve/serve.stack.e2e.test.ts | 74 ++++++++++++++----- 2 files changed, 95 insertions(+), 34 deletions(-) diff --git a/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts b/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts index f24df4b24d..02c535afb7 100644 --- a/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts @@ -1,7 +1,8 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { StackError } from "@supabase/stack/effect"; -import { Deferred, Effect, Fiber, FileSystem, Layer, Option, Path, Ref, Stream } from "effect"; +import { Deferred, Effect, Fiber, FileSystem, Layer, Option, Path, Schema, Stream } from "effect"; +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http"; import { mockOutput } from "../../../../../tests/helpers/mocks.ts"; import { mockCommandSettings, @@ -12,7 +13,36 @@ import { stackRestart } from "./restart.handler.ts"; import { destroyTestStack } from "../../../../../tests/helpers/stack-cleanup.ts"; import { stackArtifactCacheRoot } from "../../../../../tests/helpers/stack-artifacts.ts"; -const live = Layer.provideMerge(stackApiLayer, BunServices.layer); +const live = Layer.provideMerge( + stackApiLayer, + Layer.merge(BunServices.layer, FetchHttpClient.layer), +); +const MailpitInfo = Schema.Struct({ RuntimeStats: Schema.Struct({ SMTPAccepted: Schema.Finite }) }); + +// Mailpit counts accepted messages per process, so the count restarts from zero after a relaunch. +const mailpit = Effect.fn("StackRestartTest.mailpit")(function* (port: number) { + const http = yield* HttpClient.HttpClient; + const origin = `http://127.0.0.1:${port}`; + return { + send: http + .execute( + HttpClientRequest.post(`${origin}/api/v1/send`).pipe( + HttpClientRequest.bodyJsonUnsafe({ + From: { Email: "from@example.com" }, + To: [{ Email: "to@example.com" }], + Subject: "restart", + Text: "restart", + }), + ), + ) + .pipe(Effect.flatMap((response) => response.text)), + accepted: http.get(`${origin}/api/v1/info`).pipe( + Effect.flatMap((response) => response.json), + Effect.flatMap(Schema.decodeUnknownEffect(MailpitInfo)), + Effect.map((info) => info.RuntimeStats.SMTPAccepted), + ), + }; +}); const fixture = Effect.fn("StackRestartTest.fixture")(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -71,6 +101,13 @@ describe("stack restart", () => { yield* standalone.start; yield* standalone.ready; const before = yield* member.status; + const standaloneHttpPort = (yield* standalone.status).endpoints.find( + ({ name }) => name === "http", + )?.port; + if (standaloneHttpPort === undefined) return yield* Effect.die("Mail port missing"); + const standaloneMail = yield* mailpit(standaloneHttpPort); + yield* standaloneMail.send; + expect(yield* standaloneMail.accepted).toBe(1); // Both follows are subscribed once their first status arrives, before the restart. const memberFollowing = yield* Deferred.make(); @@ -84,19 +121,7 @@ describe("stack restart", () => { Stream.runDrain, Effect.forkScoped, ); - const standaloneFollowing = yield* Deferred.make(); - const standaloneLifecycles = yield* Ref.make>([]); - yield* standalone.followStatus.pipe( - Stream.tap((value) => - Ref.update(standaloneLifecycles, (seen) => [...seen, value.lifecycle]).pipe( - Effect.andThen(Deferred.succeed(standaloneFollowing, undefined)), - ), - ), - Stream.runDrain, - Effect.forkScoped, - ); yield* Deferred.await(memberFollowing); - yield* Deferred.await(standaloneFollowing); const result = yield* stackRestart(f.flags).pipe(Effect.provide(f.layer)); yield* Fiber.join(relaunched); @@ -107,7 +132,7 @@ describe("stack restart", () => { expect(after.health).toBe("healthy"); expect(after.endpoints).toEqual(before.endpoints); expect(standaloneAfter.lifecycle).toBe("running"); - expect(new Set(yield* Ref.get(standaloneLifecycles))).toEqual(new Set(["running"])); + expect(yield* standaloneMail.accepted).toBe(1); expect(f.output.stdoutText).toContain("using its saved configuration"); expect(f.telemetry.flushed).toBe(true); yield* stack.stop; diff --git a/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts b/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts index 07cdf21230..291dffc94c 100644 --- a/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts +++ b/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts @@ -1,15 +1,21 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { create as createStack } from "@supabase/stack/effect"; +import { + create as createStack, + postgres, + type Observation, + type Stack, + type StackError, +} from "@supabase/stack/effect"; import { Data, Deferred, Effect, + Fiber, FileSystem, Layer, Path, Redacted, - Ref, Schema, Stream, } from "effect"; @@ -188,23 +194,40 @@ const payload = Effect.fn("FunctionsServeE2e.payload")(function* ( const layer = Layer.mergeAll(BunServices.layer, FetchHttpClient.layer); -/** Records every lifecycle the service reports once its follow is subscribed. */ -const trackLifecycles = Effect.fn("FunctionsServeE2e.trackLifecycles")(function* (instance: { - readonly followStatus: Stream.Stream<{ readonly lifecycle: string }, unknown>; -}) { - const seen = yield* Ref.make>(new Set()); +/** + * Follows a service's statuses from the first one it delivers and returns a fiber that resolves + * with every status delivered before `barrier` matches. Statuses arrive in order, so a later + * transition the test causes proves the recorded prefix is complete. + */ +const statusesBefore = Effect.fn("FunctionsServeE2e.statusesBefore")(function* ( + instance: { readonly followStatus: Stream.Stream }, + barrier: (status: Observation) => boolean, +) { const subscribed = yield* Deferred.make(); - yield* instance.followStatus.pipe( - Stream.tap((value) => - Ref.update(seen, (lifecycles) => new Set([...lifecycles, value.lifecycle])).pipe( - Effect.andThen(Deferred.succeed(subscribed, undefined)), - ), - ), - Stream.runDrain, + const before = yield* instance.followStatus.pipe( + Stream.tap(() => Deferred.succeed(subscribed, undefined)), + Stream.takeWhile((status) => !barrier(status)), + Stream.runCollect, Effect.forkScoped, ); yield* Deferred.await(subscribed); - return seen; + return before; +}); + +/** The database process start time, which changes whenever Postgres is relaunched. */ +const postmasterStartTime = Effect.fn("FunctionsServeE2e.postmasterStartTime")(function* ( + stack: Stack, + databaseUrl: string, +) { + const decoder = new TextDecoder(); + let stdout = ""; + const result = yield* stack.commands.run(postgres.psql({ major: 17 }), { + args: ["--dbname", databaseUrl, "-tA", "--command", "SELECT pg_postmaster_start_time()"], + stdout: (bytes) => Effect.sync(() => (stdout += decoder.decode(bytes))), + stderr: () => Effect.void, + }); + expect(result.exitCode).toBe(0); + return stdout.trim(); }); describe("functions serve (stack e2e)", () => { @@ -225,7 +248,13 @@ describe("functions serve (stack e2e)", () => { expect(saved.wakeEnabled).toBe(true); const composition = yield* stack.composition.describe; const plain = yield* serve(root, home); - const lifecycles = yield* trackLifecycles(functions); + const beforeOverride = yield* statusesBefore( + functions, + (status) => + status.lifecycle === "running" && + status.config.service === "functions" && + status.config.config.env?.CUSTOM_VALUE === "overridden", + ); expect(yield* payload(apiUrl, true)).toEqual({ value: "original", anon: true, @@ -242,7 +271,6 @@ describe("functions serve (stack e2e)", () => { expect((yield* payload(apiUrl, true)).value).toBeNull(); yield* fs.writeFileString(sourcePath, source); yield* interrupt(plain); - expect(yield* Ref.get(lifecycles)).toEqual(new Set(["running"])); expect((yield* invoke(apiUrl, false)).status).toBe(401); const overridden = yield* serve(root, home, [ @@ -257,6 +285,12 @@ describe("functions serve (stack e2e)", () => { databaseUrl, }); yield* interrupt(overridden); + // The override's own restart is the only launch before its configuration is running. + const launches = (yield* Fiber.join(beforeOverride)).filter( + ({ lifecycle }, index, all) => + lifecycle === "starting" && all[index - 1]?.lifecycle !== "starting", + ); + expect(launches).toHaveLength(1); expect((yield* functions.status).config).toEqual(saved.config); expect((yield* invoke(apiUrl, false)).status).toBe(401); expect((yield* payload(apiUrl, true)).value).toBe("original"); @@ -276,7 +310,9 @@ describe("functions serve (stack e2e)", () => { const composition = yield* stack.composition.describe; const database = services.find((instance) => instance.service === "database"); if (database === undefined) return yield* Effect.die("Database missing"); - const databaseLifecycles = yield* trackLifecycles(database); + const hostDatabaseUrl = (yield* database.credentials()).databaseUrl; + if (hostDatabaseUrl === undefined) return yield* Effect.die("Database URL missing"); + const startedAt = yield* postmasterStartTime(stack, hostDatabaseUrl); const child = yield* serve(root, home); expect(yield* payload(apiUrl, true)).toEqual({ value: null, @@ -291,10 +327,10 @@ describe("functions serve (stack e2e)", () => { services.map(({ id }) => id).sort(), ); expect(yield* stack.composition.describe).toEqual(composition); - expect(yield* Ref.get(databaseLifecycles)).toEqual(new Set(["running"])); expect((yield* invoke(apiUrl, false)).status).toBe(404); const http = yield* HttpClient.HttpClient; expect((yield* http.get(`${apiUrl}/rest/v1/`)).status).toBe(200); + expect(yield* postmasterStartTime(stack, hostDatabaseUrl)).toBe(startedAt); }).pipe(Effect.provide(layer)), { timeout: 360_000 }, ); From cfcda3bd7b33714f712c49d3593b6aac0f797e29 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 12:33:24 +0200 Subject: [PATCH 119/142] refactor(stack): share the sharing-violation retry and safe-id schema Namespace modules now use the host retry helper instead of a duplicate in Capabilities, and Registry, Lease, CommandRunner and FunctionsBootstrap validate ids with one SafeId schema. --- .../stack/src/functions/FunctionsBootstrap.ts | 4 +-- packages/stack/src/host/CommandRunner.ts | 3 +- packages/stack/src/identity/SafeId.ts | 11 ++++++ packages/stack/src/namespace/Capabilities.ts | 34 +------------------ packages/stack/src/namespace/Lease.ts | 10 +++--- .../namespace/Namespace.integration.test.ts | 2 +- packages/stack/src/namespace/Publication.ts | 9 ++--- packages/stack/src/namespace/Registry.ts | 15 +++----- 8 files changed, 30 insertions(+), 58 deletions(-) create mode 100644 packages/stack/src/identity/SafeId.ts diff --git a/packages/stack/src/functions/FunctionsBootstrap.ts b/packages/stack/src/functions/FunctionsBootstrap.ts index 749849b234..b4b72c5d7b 100644 --- a/packages/stack/src/functions/FunctionsBootstrap.ts +++ b/packages/stack/src/functions/FunctionsBootstrap.ts @@ -2,6 +2,7 @@ import { Crypto, Data, Effect, FileSystem, Path, type PlatformError, Schema } fr import { contentDigestHex } from "../internal/content-digest.ts"; import { publishGeneration } from "../internal/generation-publish.ts"; import { StackIdSchema, type StackId } from "../identity/StackId.ts"; +import { isSafeId } from "../identity/SafeId.ts"; export class FunctionsBootstrapError extends Data.TaggedError("FunctionsBootstrapError")<{ readonly message: string; @@ -45,8 +46,7 @@ export const makeFunctionsBootstrapOwner = Effect.fn("FunctionsBootstrap.makeOwn yield* Schema.decodeEffect(StackIdSchema)(options.stackId).pipe( Effect.mapError((cause) => failure("Invalid stack identity", { cause })), ); - if (!/^[a-zA-Z0-9_-]+$/u.test(options.instanceId)) - return yield* failure("Invalid Functions instance identity"); + if (!isSafeId(options.instanceId)) return yield* failure("Invalid Functions instance identity"); const root = path.join(options.root, options.instanceId, "runtime", "functions"); const write = Effect.fn("FunctionsBootstrap.write")(function* (input: { diff --git a/packages/stack/src/host/CommandRunner.ts b/packages/stack/src/host/CommandRunner.ts index d0ba2abc49..216b1d9842 100644 --- a/packages/stack/src/host/CommandRunner.ts +++ b/packages/stack/src/host/CommandRunner.ts @@ -21,6 +21,7 @@ import { resolveArtifact, useNativeArtifact, } from "../Artifacts.ts"; +import { isSafeId } from "../identity/SafeId.ts"; import * as Environment from "../namespace/Environment.ts"; import { borrow } from "../namespace/Paths.ts"; import { makeContainerRuntime, type EngineTarget, type HostGateway } from "../runtime/Container.ts"; @@ -88,7 +89,7 @@ const makeCommandRunner = (options: { const crypto = yield* Crypto.Crypto; const http = yield* HttpClient.HttpClient; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - if (!/^[a-zA-Z0-9_-]+$/u.test(options.stackId)) return yield* failure("Invalid stack identity"); + if (!isSafeId(options.stackId)) return yield* failure("Invalid stack identity"); const container = options.engineTarget === undefined ? undefined diff --git a/packages/stack/src/identity/SafeId.ts b/packages/stack/src/identity/SafeId.ts new file mode 100644 index 0000000000..ad2e1b32ba --- /dev/null +++ b/packages/stack/src/identity/SafeId.ts @@ -0,0 +1,11 @@ +import { Schema } from "effect"; + +/** An identifier safe to use as a single path segment: letters, digits, `_` and `-`. */ +export const SafeId = Schema.String.pipe( + Schema.refine((value): value is string => /^[a-zA-Z0-9_-]+$/u.test(value), { + identifier: "SafeId", + message: "Expected a safe id", + }), +); + +export const isSafeId = Schema.is(SafeId); diff --git a/packages/stack/src/namespace/Capabilities.ts b/packages/stack/src/namespace/Capabilities.ts index 0e6b2c5795..8849318dd4 100644 --- a/packages/stack/src/namespace/Capabilities.ts +++ b/packages/stack/src/namespace/Capabilities.ts @@ -1,4 +1,4 @@ -import { Duration, Effect, Predicate, Schedule, Schema } from "effect"; +import { Schema } from "effect"; /** A failure from any namespace operation; `operation` names the step that failed. */ export class NamespaceError extends Schema.TaggedError()( @@ -16,35 +16,3 @@ export const namespaceError = (operation: string, cause: unknown): NamespaceErro message: cause instanceof Error ? cause.message : String(cause), cause, }); - -/** The `code` of a `PlatformError`-shaped `{ cause: { code } }`, or of any error that cause wraps. */ -export const errorCode = (error: unknown): string | undefined => { - if (!Predicate.hasProperty(error, "cause")) return undefined; - return Predicate.hasProperty(error.cause, "code") && typeof error.cause.code === "string" - ? error.cause.code - : undefined; -}; -/** Windows reports a file another process is still using or replacing as a transient violation. */ -export const isSharingViolation = - (platform: NodeJS.Platform) => - (error: unknown): boolean => - platform === "win32" && ["EPERM", "EACCES", "EBUSY"].includes(errorCode(error) ?? ""); - -/** Shared bounded backoff for retrying a transient Windows sharing violation. */ -export const transientRetrySchedule = Schedule.exponential("10 millis", 2).pipe( - Schedule.modifyDelay(({ duration }) => - Effect.succeed(Duration.min(duration, Duration.millis(100))), - ), - Schedule.upTo({ times: 12 }), -); - -/** Retries a read against `transientRetrySchedule` while it hits a sharing violation on `platform`. */ -export const retryTransientRead = - (platform?: NodeJS.Platform) => - (effect: Effect.Effect): Effect.Effect => - effect.pipe( - Effect.retry({ - schedule: transientRetrySchedule, - while: isSharingViolation(platform ?? process.platform), - }), - ); diff --git a/packages/stack/src/namespace/Lease.ts b/packages/stack/src/namespace/Lease.ts index a80362ad58..b7d55ced53 100644 --- a/packages/stack/src/namespace/Lease.ts +++ b/packages/stack/src/namespace/Lease.ts @@ -1,6 +1,8 @@ import { Effect, Exit, FileSystem, Option, Path, Schedule, Schema, Scope } from "effect"; import type { DatabaseSync } from "node:sqlite"; -import { namespaceError, retryTransientRead, type NamespaceError } from "./Capabilities.ts"; +import { isSafeId } from "../identity/SafeId.ts"; +import { retrySharingViolation } from "../internal/sharing-violation.ts"; +import { namespaceError, type NamespaceError } from "./Capabilities.ts"; import * as Publication from "./Publication.ts"; import { removeEmptyDirectory } from "./drivers/FileSystem.ts"; import { acquireLock, errcode, isBusy, isMissing, takeLock } from "./drivers/Sqlite.ts"; @@ -57,9 +59,7 @@ export const OWNER_FILE = "owner.json"; export const OWNER_LOG_FILE = "owner.log"; const checkId = (id: string): Effect.Effect => - /^[a-zA-Z0-9_-]+$/u.test(id) - ? Effect.void - : Effect.fail(namespaceError("identity", `Invalid state id: ${id}`)); + isSafeId(id) ? Effect.void : Effect.fail(namespaceError("identity", `Invalid state id: ${id}`)); export interface Options { readonly root: string; @@ -80,7 +80,7 @@ export const make = ( const leasePath = (id: string) => path.join(stackRoot(id), "owner.lock"); const ownerPath = (id: string) => path.join(stackRoot(id), OWNER_FILE); const ownerLog = (id: string) => path.join(stackRoot(id), OWNER_LOG_FILE); - const retryRead = retryTransientRead(options.platform); + const retryRead = retrySharingViolation(options.platform); /** An open file that was unlinked: SQLite IOERR_VNODE on macOS, IOERR_FSTAT on Linux. */ const isMoved = (error: NamespaceError) => diff --git a/packages/stack/src/namespace/Namespace.integration.test.ts b/packages/stack/src/namespace/Namespace.integration.test.ts index da933540d6..3d8a29acd3 100644 --- a/packages/stack/src/namespace/Namespace.integration.test.ts +++ b/packages/stack/src/namespace/Namespace.integration.test.ts @@ -21,7 +21,7 @@ import { import { existsSync } from "node:fs"; import { create, discover } from "../effect.ts"; import { destroyTestStack } from "../../tests/stack-cleanup.ts"; -import { errorCode } from "./Capabilities.ts"; +import { errorCode } from "../internal/sharing-violation.ts"; import * as StackNamespace from "../StackNamespace.ts"; import * as Paths from "./Paths.ts"; diff --git a/packages/stack/src/namespace/Publication.ts b/packages/stack/src/namespace/Publication.ts index c1e2d57826..6b3c2558ec 100644 --- a/packages/stack/src/namespace/Publication.ts +++ b/packages/stack/src/namespace/Publication.ts @@ -1,5 +1,6 @@ import { Effect, FileSystem, Path } from "effect"; -import { isSharingViolation, namespaceError, transientRetrySchedule } from "./Capabilities.ts"; +import { retrySharingViolation } from "../internal/sharing-violation.ts"; +import { namespaceError } from "./Capabilities.ts"; import * as Drivers from "./drivers/FileSystem.ts"; export interface PublishOptions { @@ -28,11 +29,7 @@ export const publish = Effect.fn("Namespace.publish")(function* ( directory, `.${path.basename(options.target)}.${process.pid}-${++counter}.tmp`, ); - const platform = options.platform ?? process.platform; - const retry = (effect: Effect.Effect) => - effect.pipe( - Effect.retry({ schedule: transientRetrySchedule, while: isSharingViolation(platform) }), - ); + const retry = retrySharingViolation(options.platform); yield* Effect.acquireUseRelease( Effect.succeed(staging), () => diff --git a/packages/stack/src/namespace/Registry.ts b/packages/stack/src/namespace/Registry.ts index ed8d340ea9..38aa0b5768 100644 --- a/packages/stack/src/namespace/Registry.ts +++ b/packages/stack/src/namespace/Registry.ts @@ -1,7 +1,9 @@ import { Clock, Effect, Exit, FileSystem, Option, Path, Schedule, Schema, Scope } from "effect"; import { CompositionConfig } from "../Orchestrator.ts"; import { ServiceCreation } from "../services/Catalog.ts"; -import { namespaceError, retryTransientRead, type NamespaceError } from "./Capabilities.ts"; +import { SafeId } from "../identity/SafeId.ts"; +import { retrySharingViolation } from "../internal/sharing-violation.ts"; +import { namespaceError, type NamespaceError } from "./Capabilities.ts"; import { OWNER_FILE, OWNER_LOG_FILE } from "./Lease.ts"; import { stackLogsRoot } from "./Paths.ts"; import * as Publication from "./Publication.ts"; @@ -11,13 +13,6 @@ import { acquireLock, isBusy, takeLock } from "./drivers/Sqlite.ts"; /** How long a destroyed stack's empty directories stay, so a same-id restart finds them. */ const DESTROYED_RETENTION_MILLIS = 10 * 60 * 1000; -const SafeId = Schema.String.pipe( - Schema.refine((value): value is string => /^[a-zA-Z0-9_-]+$/u.test(value), { - identifier: "SafeStateId", - message: "Expected a safe state id", - }), -); - const SavedInstance = Schema.Struct({ id: SafeId, creation: Schema.toCodecJson(ServiceCreation), @@ -131,7 +126,7 @@ export const make = ( const statePath = (id: string) => path.join(stackRoot(id), "state.json"); const retryRead = (effect: Effect.Effect) => effect.pipe( - retryTransientRead(options.platform), + retrySharingViolation(options.platform), Effect.mapError((cause) => namespaceError("read", cause)), ); const read = Effect.fn("Namespace.Registry.read")(function* (id: string) { @@ -184,7 +179,7 @@ export const make = ( yield* checkId(id); // Logs go before the state file, so a stack whose logs remain stays listed for another removal. yield* fs.remove(stackLogsRoot(path, root, id), { recursive: true, force: true }).pipe( - retryTransientRead(options.platform), + retrySharingViolation(options.platform), Effect.mapError((cause) => namespaceError("remove", cause)), ); for (const file of [ From 57c8fd2baa98534695162d0ff9296def6a79511a Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 12:43:32 +0200 Subject: [PATCH 120/142] ci(repo): drop the temporary stack-architecture branch workflow --- .github/workflows/branch-ci.yml | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 .github/workflows/branch-ci.yml diff --git a/.github/workflows/branch-ci.yml b/.github/workflows/branch-ci.yml deleted file mode 100644 index cacc708e15..0000000000 --- a/.github/workflows/branch-ci.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: branch-ci - -# Runs the Test suite on pushes to stack-architecture branches, which have no PR yet. -on: - push: - branches: - - stack-architecture - - stack-architecture/** - -permissions: - actions: read - contents: read - -concurrency: - group: branch-ci-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - name: Test - uses: ./.github/workflows/test.yml - with: - force: true - secrets: - DF_FIREWALL_TOKEN: ${{ secrets.DF_FIREWALL_TOKEN }} From 967119161d384df3d351fb49f186d5af2f3e2a45 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 13:57:11 +0200 Subject: [PATCH 121/142] fix(stack): keep launch crashes across retried cleanup and idle unhealthy lazy services --- packages/stack/src/Lifecycle.ts | 19 ++++--- packages/stack/src/Lifecycle.unit.test.ts | 67 +++++++++++++++++++++++ 2 files changed, 79 insertions(+), 7 deletions(-) diff --git a/packages/stack/src/Lifecycle.ts b/packages/stack/src/Lifecycle.ts index 6ec86c8cfc..f4716c31c7 100644 --- a/packages/stack/src/Lifecycle.ts +++ b/packages/stack/src/Lifecycle.ts @@ -781,15 +781,16 @@ const applyEvent = ( cause, commands, ); + const crash = + service.phase._tag === "Stopping" + ? service.phase.crash + : failure === undefined + ? undefined + : { cause: failure }; return [ setService(dependentsFailed, id, (s) => ({ ...failAllWaiters(s, id, message, cause, commands), - phase: Phase.Stopping({ - generation, - ...(service.phase._tag === "Stopping" && service.phase.crash !== undefined - ? { crash: service.phase.crash } - : {}), - }), + phase: Phase.Stopping({ generation, ...(crash === undefined ? {} : { crash }) }), breaker: failure === undefined || service.phase._tag === "Stopping" || @@ -1197,7 +1198,11 @@ const settle = ( // arm/clear decision must never be stranded by a readiness loss that follows it, or the // service could never become idle-eligible again once readiness recovers. next = setService(next, id, (s) => ({ ...s, idleArmedEpoch: undefined })); - } else if (!demand && settled.phase.ready && settled.idleArmedEpoch === undefined) { + } else if ( + !demand && + (settled.phase.ready || settled.readinessFailure !== undefined) && + settled.idleArmedEpoch === undefined + ) { const epoch = settled.idleEpoch + 1; emitted.push( LifecycleCommand.ArmIdleTimer({ diff --git a/packages/stack/src/Lifecycle.unit.test.ts b/packages/stack/src/Lifecycle.unit.test.ts index 85f9123f68..1ac8054522 100644 --- a/packages/stack/src/Lifecycle.unit.test.ts +++ b/packages/stack/src/Lifecycle.unit.test.ts @@ -779,6 +779,44 @@ describe("Lifecycle reducer", () => { }); }); + it("ends Failed without relaunching an eager service whose launch cleanup was retained and retried by traffic", () => { + let state = initialState( + makeGraph([{ id: "database", activation: "eager", prerequisites: [] }]), + ); + ({ state } = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "database" }), now: 0 }, + { + event: LifecycleEvent.LaunchFailed({ id: "database", generation: 1, cause: "boom" }), + now: 1, + }, + { + event: LifecycleEvent.StopFailed({ + id: "database", + generation: 1, + cause: "busy", + failure: "boom", + }), + now: 2, + }, + ])); + + let commandsByStep: ReadonlyArray>; + ({ state, commandsByStep } = run(state, [ + { event: open("database", 1), now: 3 }, + { event: LifecycleEvent.WaiterCancelled({ id: "database", waiterId: 1 }), now: 4 }, + { + event: LifecycleEvent.Exited({ id: "database", generation: 1, requested: true }), + now: 5, + }, + ])); + expect(tagsOf(commandsByStep[0])).toContain("Stop"); + expect(commandsByStep[2]).toEqual([]); + expect(state.services.get("database")?.phase).toMatchObject({ + _tag: "Failed", + cause: "boom", + }); + }); + it("ends Stopped when an explicit stop arrives during the crash's cleanup", () => { let state = initialState(makeGraph([lazy("api")])); ({ state } = run(state, [ @@ -1334,6 +1372,35 @@ describe("Lifecycle reducer", () => { expect(state.services.get("api")?.idleArmedEpoch).toBe(2); }); + it("idles an unhealthy lazy service without demand and releases its prerequisites", () => { + let state = initialState( + makeGraph([lazy("database"), lazy("api", { prerequisites: ["database"] })]), + ); + let commandsByStep: ReadonlyArray>; + ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "database", generation: 1 }), now: 1 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 2 }, + { event: LifecycleEvent.ReadinessLost({ id: "api", generation: 1, cause: "503" }), now: 3 }, + ])); + expect(state.services.get("api")?.idleArmedEpoch).toBeUndefined(); + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 4 }, + ])); + expect(tagsOf(commandsByStep[0])).toContain("ArmIdleTimer"); + const epoch = state.services.get("api")?.idleArmedEpoch ?? -1; + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.IdleElapsed({ id: "api", generation: 1, epoch }), now: 1_004 }, + { event: LifecycleEvent.Exited({ id: "api", generation: 1, requested: true }), now: 1_005 }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["Stop"]); + expect(state.services.get("api")?.phase).toEqual({ _tag: "Stopped" }); + expect(tagsOf(commandsByStep[1])).toEqual(["ArmIdleTimer"]); + expect(state.services.get("database")?.idleArmedEpoch).toBe(1); + }); + it("fails only the waiters awaiting a failed readiness check, keeping the session and inspector waiters", () => { const graph = makeGraph([lazy("database"), lazy("api", { prerequisites: ["database"] })]); let state = initialState(graph); From e2155af33cc58972a9e24cba7a14a852dd0be15b Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 13:57:25 +0200 Subject: [PATCH 122/142] fix(stack): never follow symlinks in the artifact cache sweep --- .../stack/src/preparation/ArtifactStore.ts | 8 +++++ .../preparation/artifacts.integration.test.ts | 31 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/packages/stack/src/preparation/ArtifactStore.ts b/packages/stack/src/preparation/ArtifactStore.ts index 68719d266c..4702632c74 100644 --- a/packages/stack/src/preparation/ArtifactStore.ts +++ b/packages/stack/src/preparation/ArtifactStore.ts @@ -613,6 +613,14 @@ const walkCache = ( const generations: Array = []; const stagingRoots: Array = []; for (const name of names) { + // readLink succeeds only on a symlink; a link is never a cache entry, so it is left alone. + if ( + yield* fs.readLink(path.join(root, name)).pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + ) + ) + continue; if (name === STAGING_DIR_NAME) { stagingRoots.push(path.join(root, name)); continue; diff --git a/packages/stack/src/preparation/artifacts.integration.test.ts b/packages/stack/src/preparation/artifacts.integration.test.ts index 8530facd6e..e1313ce8ad 100644 --- a/packages/stack/src/preparation/artifacts.integration.test.ts +++ b/packages/stack/src/preparation/artifacts.integration.test.ts @@ -713,6 +713,37 @@ describe("lock-guarded staging", () => { 20_000, ); + it.live.skipIf(process.platform === "win32")( + "never follows a symlinked staging root or cache subdirectory out of the cache", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-sweep-link-", + }); + const outside = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-sweep-outside-", + }); + const key = "database/postgres-sweep-link"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + yield* store.prepare({ ...request, key }); + yield* fs.makeDirectory(`${outside}/token`, { recursive: true }); + yield* fs.writeFileString(`${outside}/token/precious`, "keep"); + yield* fs.writeFileString(`${outside}/precious`, "keep"); + yield* fs.remove(`${root}/${key}/.staging`, { recursive: true, force: true }); + yield* fs.symlink(outside, `${root}/${key}/.staging`); + yield* fs.symlink(outside, `${root}/linked-subdir`); + + yield* triggerSweep(root); + + expect(yield* fs.exists(`${outside}/token/precious`)).toBe(true); + expect(yield* fs.exists(`${outside}/precious`)).toBe(true); + }), + ), + 20_000, + ); + it.live( "leaves a staging lock file that has no directory yet for the preparer that is about to take it", () => From 7157b3b27ee42536b7ef1c0f92edbabf4bcbba77 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 13:57:38 +0200 Subject: [PATCH 123/142] docs(stack): record the native Postgres step-down in ADR 0028 and fix Podman test coverage --- .github/workflows/test.yml | 3 +- apps/cli/docs/analytics.md | 2 +- .../stack/stack.shared.integration.test.ts | 18 +++++- ...5-ephemeral-postgres-for-schema-tooling.md | 24 ++++---- ...028-native-postgres-privilege-step-down.md | 57 +++++++++++++++++++ docs/adr/README.md | 1 + 6 files changed, 87 insertions(+), 18 deletions(-) create mode 100644 docs/adr/0028-native-postgres-privilege-step-down.md diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f8958f7d39..76cd0c8efc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -402,10 +402,9 @@ jobs: src/services/Rest.integration.test.ts src/services/AuthStorage.integration.test.ts src/services/StorageGateway.integration.test.ts - src/services/AnalyticsVectorImgproxy.integration.test.ts + src/services/AnalyticsImgproxy.integration.test.ts src/services/Mail.integration.test.ts src/services/Pooler.integration.test.ts - src/services/Vector.integration.test.ts tests/docker-volume-run.integration.test.ts --passWithNoTests=false diff --git a/apps/cli/docs/analytics.md b/apps/cli/docs/analytics.md index 61472e0b30..225713f0fc 100644 --- a/apps/cli/docs/analytics.md +++ b/apps/cli/docs/analytics.md @@ -90,7 +90,7 @@ Local-runtime commands also carry properties that split error rates by runtime: - `stack_backend` (`legacy` or `stack`) on commands routed by `experimental.stack`; see [backend selection](stack-commands.md) -- `stack_runtime` (`docker` or `native`): the runtime of the stack a stack-backend +- `stack_runtime` (`docker`, `podman`, or `native`): the runtime of the stack a stack-backend command looked up, which for `--db-url` may be the project stack even when the URL targets another database - `orioledb` (boolean) once the command resolves local database config, `true` when the project diff --git a/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts index 7550f1b17c..ffa34116e0 100644 --- a/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts @@ -18,7 +18,7 @@ type TargetInput = { readonly projectRoot: string; readonly name?: string; readonly id?: string; - readonly runtime: "auto" | "docker" | "native"; + readonly runtime: "auto" | "docker" | "podman" | "native"; }; const workspace = Effect.gen(function* () { @@ -186,6 +186,22 @@ describe("stack target resolver", () => { }).pipe(Effect.provide(BunServices.layer)), ); + it.live("carries an explicit Podman runtime onto a new stack target and records it", () => + Effect.gen(function* () { + const { root, resolve } = yield* workspace; + const recorded = yield* Ref.make({}); + + const target = yield* resolve({ projectRoot: root, runtime: "podman" }).pipe( + Effect.provideService(CommandTelemetryAttributes, { + record: (values) => Ref.update(recorded, (current) => ({ ...current, ...values })), + }), + ); + + expect(target).toEqual({ projectRoot: root, runtime: "podman", hostRunning: false }); + expect((yield* Ref.get(recorded)).stack_runtime).toBe("podman"); + }).pipe(Effect.provide(BunServices.layer)), + ); + it.live("returns the canonical project root for a new stack reached through a symlink", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md b/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md index a3415da6ee..a04654aef8 100644 --- a/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md +++ b/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md @@ -1,6 +1,6 @@ # 0025. Ephemeral Postgres for schema tooling -**Status**: superseded by throwaway stack shadow stacks and [database snapshots](../../packages/stack/ARCHITECTURE.md#snapshots-belong-to-the-database-instance) — `EphemeralPostgres` was never built; see [`stack-shadow.ts`](../../apps/cli/src/command-internal/stack-shadow.ts) +**Status**: superseded by throwaway stack shadow stacks and [database snapshots](../../packages/stack/ARCHITECTURE.md#snapshots-belong-to-the-database-instance) — `EphemeralPostgres` was never built; see [`stack-shadow.ts`](../../apps/cli/src/command-internal/stack-shadow.ts); runtime auto-selection and native-as-root handling amended by [ADR 0028](0028-native-postgres-privilege-step-down.md) **Date**: 2026-09-09 ## Problem Statement @@ -124,17 +124,13 @@ whether **this start created the stack identity** (first create / `unconfigured` ### Default runtime and native-as-root -Auto-selecting a **new** identity probes the Docker engine (not only `docker --version`), then -Podman. A reachable engine persists its runtime. When neither is reachable, **native** is persisted -where native artifacts exist. Skipping Docker prints a notice that the chosen runtime is saved with -the stack and that destroy-and-recreate (or a new `--stack` name) is required to get Docker later. -Persisted runtime never flips. Explicit `--runtime docker` or `--runtime podman` still requires a -reachable engine. +Auto-selecting a **new** identity probes the Docker daemon (not only `docker --version`). A live +daemon persists Docker. A present client with a dead daemon persists **native** and prints a +notice that destroy-and-recreate (or a new `--stack` name) is required to get Docker later. +Persisted runtime never flips. Explicit `--runtime docker` still requires a live daemon. -Native Postgres refuses to run as uid 0 (`initdb` refuses root), so when the stack runs as root -only the PostgreSQL process steps down to an unprivileged user: one named by -`SUPABASE_NATIVE_POSTGRES_USER`, or the account of a detected agent sandbox. Otherwise use -`--runtime docker`. +Native Postgres is refused when the process uid is 0 (`initdb` refuses root). There is no +uid-drop. Use `--runtime docker`. ### Optional catalog downloads @@ -153,7 +149,7 @@ leaving schema policy in the CLI. ### Positive -- Native and container shadows share one API and the same slim baseline as `stack start`. +- Native and Docker/Podman shadows share one API and the same slim baseline as `stack start`. - Schema commands can target a running project stack through `credentials()` when the flag is on. `credentials().database` is available whenever the database listener is assigned, including when Auth is disabled. `credentials().api` is absent when Auth is off. Overlay and `--local` keep @@ -170,8 +166,8 @@ leaving schema policy in the CLI. - A failed first live setup after the engine is running is stuck until `db reset`, same as Compose. A failed cold launch that never reached running retries first-create. - Windows native dump/test/squash need a working Docker client even though Postgres itself is native. -- Native stacks as uid 0 need `SUPABASE_NATIVE_POSTGRES_USER` or a detected agent sandbox; otherwise Docker (or a non-root user) is required. -- Auto-selected native or Podman after an unreachable Docker engine is sticky until destroy or a new stack name. +- Native stacks as uid 0 cannot start; Docker (or a non-root user) is required. +- Auto-selected native after a dead Docker daemon is sticky until destroy or a new stack name. ## Alternatives Considered diff --git a/docs/adr/0028-native-postgres-privilege-step-down.md b/docs/adr/0028-native-postgres-privilege-step-down.md new file mode 100644 index 0000000000..b616c5bb75 --- /dev/null +++ b/docs/adr/0028-native-postgres-privilege-step-down.md @@ -0,0 +1,57 @@ +# 0028. Native Postgres privilege step-down + +**Status**: proposed +**Date**: 2026-10-07 + +## Problem Statement + +[ADR 0025](0025-ephemeral-postgres-for-schema-tooling.md) refuses native Postgres when the process +uid is 0, because `initdb` refuses to run as root, and tells users to pick `--runtime docker`. +Containers and agent sandboxes commonly run the CLI as root without a container engine, so that +rule leaves no working runtime there. Automatic runtime selection also only chose between Docker +and native, ignoring Podman. + +## Decision + +- When the stack runs as root, only the PostgreSQL process steps down to an unprivileged user. The + user is named by `SUPABASE_NATIVE_POSTGRES_USER`, or is the account of a detected agent sandbox. + The CLI itself keeps running as root and hands the instance data, root key, and socket directory + to that user. Without either, native startup still refuses root. +- Auto-selecting the runtime of a new stack probes Docker, then Podman, then falls back to native + where native artifacts exist. An explicit `--runtime` has no fallback, and a saved runtime never + flips. + +## Rationale + +Stepping down only the process that requires it keeps the rest of the stack unchanged and avoids a +second privilege model. Probing Podman before native means a machine with either engine gets +containers, as it does when started explicitly. + +## Consequences + +### Positive + +- Native stacks start in root-only environments when a user is named or a sandbox is detected. +- Podman-only machines get a container runtime by default. + +### Negative + +- Root-owned native stacks need ownership changes on stack state and traverse permission on parent + directories, described in the `stack start` side effects. +- A runtime chosen automatically after skipping Docker is sticky until the stack is destroyed or a + new stack name is used. + +## Alternatives Considered + +1. **Keep refusing root**: leaves root-only environments without a runtime. +2. **Run the whole stack as the unprivileged user**: requires re-executing the CLI and splits + state ownership across users. + +## Related Decisions + +- [ADR 0025](0025-ephemeral-postgres-for-schema-tooling.md): the native-as-root and default + runtime rules this decision amends. + +## See Also + +- [`stack start` side effects](../../apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md) diff --git a/docs/adr/README.md b/docs/adr/README.md index efbb6f77f1..ed20bcd7df 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -69,6 +69,7 @@ When an ADR becomes outdated, mark it as `deprecated` or reference the supersedi | 0025 | [Ephemeral Postgres for Schema Tooling](0025-ephemeral-postgres-for-schema-tooling.md) | superseded | | 0026 | [Slim Image and Native Artifact Mirrors](0026-slim-artifact-mirrors.md) | proposed | | 0027 | [CLI Tracing Conventions](0027-cli-tracing-conventions.md) | proposed | +| 0028 | [Native Postgres Privilege Step-Down](0028-native-postgres-privilege-step-down.md) | proposed | ## Template From 1d74624ceef15a438ae2857bdf88af3cf6ab1134 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 14:20:20 +0200 Subject: [PATCH 124/142] fix(stack): end lost ownership without destroying and refuse a symlinked data root --- packages/stack/src/Owner.integration.test.ts | 32 ++++ packages/stack/src/Owner.ts | 15 +- .../stack/src/StackHost.integration.test.ts | 167 +++++++++++------- packages/stack/src/StackHost.ts | 124 ++++++------- packages/stack/src/Sweep.integration.test.ts | 34 +++- packages/stack/src/Sweep.ts | 2 +- packages/stack/src/namespace/Lease.ts | 29 ++- .../namespace/Namespace.integration.test.ts | 33 ++++ .../src/namespace/Paths.integration.test.ts | 24 ++- packages/stack/src/namespace/Paths.ts | 55 ++++-- 10 files changed, 347 insertions(+), 168 deletions(-) diff --git a/packages/stack/src/Owner.integration.test.ts b/packages/stack/src/Owner.integration.test.ts index 3bfeef2ef1..ca72b84be8 100644 --- a/packages/stack/src/Owner.integration.test.ts +++ b/packages/stack/src/Owner.integration.test.ts @@ -893,6 +893,38 @@ it.live("destroying a stack leaves its data root empty after an owner was killed ), ); +it.live("refuses to destroy a stack whose data directory is a symlink and keeps its target", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-symlinked-data-" }); + const stack = initial(`owner-symlinked-data-${randomUUID().slice(0, 8)}`); + const state = yield* stateFor(`${root}/state`); + yield* state.save(stack); + const outside = `${root}/outside`; + yield* fs.makeDirectory(outside); + yield* fs.writeFileString(`${outside}/precious.txt`, "keep"); + const data = `${root}/state/${stack.id}/data`; + yield* fs.symlink(outside, data); + const owner = yield* ownerFor({ saved: stack, state, root: data, cacheRoot }); + + const failure = yield* owner.namespace.destroy.pipe(Effect.flip); + + expect(failure.message).toContain("symlink"); + expect(yield* fs.readFileString(`${outside}/precious.txt`)).toBe("keep"); + expect(yield* state.read(stack.id), "the registration stays for a retry").toBeDefined(); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), +); + it.effect("refuses to generate credentials for a stack whose saved instances consume them", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index f5135ba316..95dadc6ff3 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -64,6 +64,7 @@ import type { CatalogError } from "./services/Recipe.ts"; import * as Container from "./runtime/Container.ts"; import { projectSegmentFor } from "./identity/Identity.ts"; import { stackError, type OwnerRpc } from "./Rpc.ts"; +import { confirmStackDataRoot } from "./namespace/Paths.ts"; import * as StackNamespace from "./StackNamespace.ts"; import type { SavedStack, StackCredentials, StackKeysInput } from "./StackNamespace.ts"; import { makeDockerHelperRegistry } from "./storage/DockerHelperRegistry.ts"; @@ -142,6 +143,12 @@ export interface Interface { * containers, native socket directories and saved state. */ readonly destroy: Effect.Effect; + /** + * Stops what this owner launched once in-flight definition changes settle, for an owner whose + * claim on the stack ended: containers, data, ports and saved state may now belong to a + * successor and stay untouched. + */ + readonly release: Effect.Effect; }; } @@ -874,7 +881,9 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ), // Instance teardown retains port rows until the registration is gone; the release after // that is best-effort because `isGone` reclaims a gone stack's rows lazily. - destroy: orchestrator.destroyNamespace.pipe( + destroy: confirmStackDataRoot(options.state.root, stackId).pipe( + Effect.provideContext(services), + Effect.andThen(orchestrator.destroyNamespace), Effect.andThen(sweep), Effect.andThen(removeDataRootEntries), Effect.andThen(logStore.close), @@ -889,6 +898,10 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { definitionGate.withPermits(1), Effect.withSpan("Owner.destroyNamespace"), ), + release: orchestrator.stopNamespace.pipe( + definitionGate.withPermits(1), + Effect.withSpan("Owner.releaseNamespace"), + ), }, } satisfies Interface; }); diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts index 6145cdb1b2..8ac184fac6 100644 --- a/packages/stack/src/StackHost.integration.test.ts +++ b/packages/stack/src/StackHost.integration.test.ts @@ -20,6 +20,7 @@ import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"; import * as HttpClient from "effect/unstable/http/HttpClient"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- integration observes exact listener closure. import * as Net from "node:net"; +import { fileURLToPath } from "node:url"; import { launchHost, ownerAuthorization, @@ -33,11 +34,15 @@ import { OrchestratorError } from "./Orchestrator.ts"; import { CommandEvent, StackError } from "./Rpc.ts"; import * as StackNamespace from "./StackNamespace.ts"; import { bindControl, makeRuntime } from "./StackHost.ts"; -import { shutdownOwner } from "../tests/owner.ts"; +import { shutdownOwner, watchLeaseRelease } from "../tests/owner.ts"; import { postgres } from "./Commands.ts"; import * as CommandRunner from "./host/CommandRunner.ts"; import { testArtifactCacheRoot } from "../tests/artifact-cache.ts"; +const shortRegistrationPollFixture = fileURLToPath( + new URL("../tests/short-registration-poll-fixture.ts", import.meta.url), +); + class HostTestError extends Data.TaggedError("HostTestError")<{ readonly message: string }> {} const stateFor = (root: string) => @@ -313,72 +318,6 @@ it.live("stops serving after a failed shutdown and keeps reporting that failure" ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); -it.live( - "closes admission and confirms cleanup when abandonment claims ownership after a failing stop settles", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-host-abandon-after-stop-", - }); - const state = yield* stateFor(`${root}/state`); - const saved = { - id: "stack", - runtime: "native" as const, - identity: { - projectRoot: root, - branchContext: "main", - stackName: "host-abandon-after-stop", - }, - instances: [], - lifetime: "detached" as const, - composition: { members: [], dependencies: [] }, - }; - yield* state.save(saved); - const owner = yield* ownerFor({ - saved, - state, - root: `${root}/data`, - cacheRoot: testArtifactCacheRoot, - }); - const stopGate = yield* Deferred.make(); - const destroyEntered = yield* Deferred.make(); - const destroyGate = yield* Deferred.make(); - const failedOwner = { - ...owner, - namespace: { - ...owner.namespace, - stop: Deferred.await(stopGate).pipe( - Effect.andThen( - Effect.fail(new OrchestratorError({ operation: "stop", message: "stop failed" })), - ), - ), - destroy: Deferred.succeed(destroyEntered, undefined).pipe( - Effect.andThen(Deferred.await(destroyGate)), - Effect.andThen(owner.namespace.destroy), - ), - }, - }; - const { runtime } = yield* inProcessRuntime(failedOwner, state, root); - const stopFiber = yield* Effect.forkScoped(runtime.shutdown(false)); - // Abandon is requested while the stop is still in flight, never after it has settled: - // the one shutdown pipeline waits for it, instead of racing its own cleanup. - const abandonFiber = yield* Effect.forkScoped(runtime.abandon); - yield* Deferred.succeed(stopGate, undefined); - expect(Exit.isFailure(yield* Fiber.join(stopFiber).pipe(Effect.exit))).toBe(true); - yield* Deferred.await(destroyEntered).pipe(Effect.timeout("2 seconds")); - expect( - yield* Deferred.isDone(owner.draining), - "admission stays closed while abandonment cleans up", - ).toBe(true); - yield* Deferred.succeed(destroyGate, undefined); - yield* Fiber.join(abandonFiber).pipe(Effect.timeout("2 seconds")); - yield* Deferred.await(runtime.exit).pipe(Effect.timeout("2 seconds")); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), -); - it.live("reports a destroy failure and the owner exits", () => Effect.scoped( Effect.gen(function* () { @@ -1084,3 +1023,97 @@ it.live("destroys a stack only after an abandoned composition settles", () => }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); + +const endedOwnership = (prefix: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix }); + const stateRoot = `${root}/state`; + const state = yield* stateFor(stateRoot); + const saved = { + id: "stack", + runtime: "native" as const, + identity: { projectRoot: root, branchContext: "main", stackName: "host-ownership" }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + }; + yield* state.save(saved); + // The shortened poll interval comes only from this dedicated test entrypoint. + const launch = launchHost(state, { + stateRoot, + cacheRoot: testArtifactCacheRoot, + stackId: saved.id, + entrypoint: shortRegistrationPollFixture, + }); + const access = yield* launch; + yield* Effect.addFinalizer(() => + shutdownOwner(access, true).pipe( + Effect.ignore, + Effect.andThen( + waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe(Effect.ignore), + ), + ), + ); + return { fs, stateRoot, state, saved, launch, access }; + }); + +it.live.skipIf(process.platform === "win32")( + "exits when its stack directory is deleted and leaves a restarted stack's owner discoverable", + () => + Effect.scoped( + Effect.gen(function* () { + const { fs, stateRoot, state, saved, launch, access } = + yield* endedOwnership("stack-host-dir-deleted-"); + const exited = yield* watchLeaseRelease(stateRoot, saved.id); + yield* fs.remove(`${stateRoot}/${saved.id}`, { recursive: true }); + yield* state.save(saved); + const successor = yield* launch; + yield* Effect.addFinalizer(() => + shutdownOwner(successor, true).pipe( + Effect.ignore, + Effect.andThen( + waitForOwnerExit(successor.endpoint.pid, ownerExitProbe(fs)).pipe(Effect.ignore), + ), + ), + ); + yield* exited; + yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.timeout("10 seconds"), + ); + + expect(successor.endpoint.pid, "the restart started its own owner").not.toBe( + access.endpoint.pid, + ); + expect((yield* state.readHolder(saved.id))?.role === "owner").toBe(true); + expect((yield* launch).endpoint.pid, "the successor is still discoverable").toBe( + successor.endpoint.pid, + ); + yield* shutdownOwner(successor, false); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 60_000 }, +); + +it.live.skipIf(process.platform === "win32")( + "exits without deleting data when only its registration is deleted", + () => + Effect.scoped( + Effect.gen(function* () { + const { fs, stateRoot, saved, access } = yield* endedOwnership( + "stack-host-registration-deleted-", + ); + const kept = `${stateRoot}/${saved.id}/data/kept.txt`; + yield* fs.writeFileString(kept, "data"); + const exited = yield* watchLeaseRelease(stateRoot, saved.id); + yield* fs.remove(`${stateRoot}/${saved.id}/state.json`); + yield* exited; + yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.timeout("10 seconds"), + ); + + expect(yield* fs.readFileString(kept)).toBe("data"); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 60_000 }, +); diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index 2c6386ce0a..51eb4d2e39 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -173,10 +173,11 @@ export interface StackHostRuntime { response?: ReturnType, ) => Effect.Effect; /** - * Ends ownership after a confirmed-gone registration with one best-effort destroy: it joins a - * running destroy and runs after a running stop settles. Failures are logged, never raised. + * Ends ownership that was lost to a deleted registration or a replaced lease file: stops only + * what this owner launched and leaves data, ports, registration and any successor's containers + * alone. It joins a shutdown already running. Failures are logged, never raised. */ - readonly abandon: Effect.Effect; + readonly release: Effect.Effect; readonly exit: Deferred.Deferred; } @@ -222,7 +223,7 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( }); const exit = yield* Deferred.make(); const gate = yield* Semaphore.make(1); - type ShutdownMode = "stop" | "destroy"; + type ShutdownMode = "stop" | "destroy" | "release"; const current = yield* Ref.make< { readonly mode: ShutdownMode; readonly fiber: Fiber.Fiber } | undefined >(undefined); @@ -235,9 +236,9 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( scope, ); // Admits `mode`'s cleanup when nothing else is in flight: begins draining, forks the cleanup - // and tracks it as `current`. Draining never reverses: a failed stop or destroy leaves its - // failed fiber in `current` (repeat calls rejoin the same failure), and the owner exits - // once the response has been delivered; only abandonment may replace a settled stop. + // and tracks it as `current`. Draining never reverses: a failed shutdown leaves its failed + // fiber in `current` (repeat calls rejoin the same failure), and the owner exits once the + // response has been delivered. const begin = ( mode: ShutdownMode, response?: ReturnType, @@ -266,7 +267,7 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( const body = Effect.gen(function* () { yield* attachments.stopAll; yield* runner.cleanup; - yield* mode === "stop" ? owner.namespace.stop : owner.namespace.destroy; + yield* owner.namespace[mode]; }).pipe( Effect.onError(() => finish), Effect.andThen(finish), @@ -277,47 +278,29 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( return fiber; }); - // The one shutdown pipeline: `stop` and `destroy` are claimed through the same gate and - // `current` record. A repeat request rejoins the shutdown in flight (a stop after a - // destroy, or the same mode again), and a destroy after a stop is rejected, except for - // `afterStop`: abandonment waits for the stop to settle, whatever its outcome, then claims - // its destroy fresh. + // The one shutdown pipeline: every mode is claimed through the same gate and `current` + // record. A repeat request rejoins the shutdown in flight (a stop after a destroy, or the + // same mode again), and a destroy after a stop is rejected. const claim = ( mode: ShutdownMode, response?: ReturnType, - afterStop = false, ): Effect.Effect => - Effect.gen(function* () { - while (true) { - const decision = yield* gate.withPermits(1)( - Effect.uninterruptibleMask(() => - Effect.gen(function* () { - const existing = yield* Ref.get(current); - if (existing === undefined) { - const fiber = yield* begin(mode, response); - return { _tag: "run", fiber } as const; - } - if (existing.mode === "stop" && mode === "destroy") { - if (afterStop) return { _tag: "await", fiber: existing.fiber } as const; - return yield* new StackError({ - operation: "shutdown", - message: "Shutdown mode is already selected", - }); - } - return { _tag: "run", fiber: existing.fiber } as const; - }), - ), - ); - if (decision._tag === "run") { - yield* Fiber.join(decision.fiber); - return; - } - yield* Fiber.await(decision.fiber); - yield* gate.withPermits(1)( - Ref.update(current, (value) => (value?.fiber === decision.fiber ? undefined : value)), - ); - } - }); + gate + .withPermits(1)( + Effect.uninterruptibleMask(() => + Effect.gen(function* () { + const existing = yield* Ref.get(current); + if (existing === undefined) return yield* begin(mode, response); + if (existing.mode === "stop" && mode === "destroy") + return yield* new StackError({ + operation: "shutdown", + message: "Shutdown mode is already selected", + }); + return existing.fiber; + }), + ), + ) + .pipe(Effect.flatMap(Fiber.join)); const shutdown = ( destroy: boolean, response?: ReturnType, @@ -325,9 +308,9 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( claim(destroy ? "destroy" : "stop", response).pipe( Effect.mapError((cause) => stackError("shutdown", cause)), ); - // A registration-loss poll drives this, never an RPC caller, so there is no response to watch. - const abandon = claim("destroy", undefined, true).pipe( - Effect.tapCause((cause) => Effect.logError("Abandoned stack destroy failed", cause)), + // The ownership poll drives this, never an RPC caller, so there is no response to watch. + const release = claim("release").pipe( + Effect.tapCause((cause) => Effect.logError("Releasing the stack failed", cause)), Effect.ignore, ); const handlers = StackRpc.of({ @@ -384,49 +367,46 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( access, serve, shutdown, - abandon, + release, closeConnections, exit, }; }), ); -type HostEvent = "SIGTERM" | "SIGINT" | "creator-gone" | "registration-gone"; +type HostEvent = "SIGTERM" | "SIGINT" | "creator-gone" | "ownership-ended"; /** - * True only on a confirmed ENOENT (`state.read` returning `undefined`); any other error - * (permissions, an unmounted root mid-read) keeps the owner running, so it reads as "not - * confirmed gone" rather than abandonment. + * Ownership holds while the stack is registered and the lease path still names the file this + * owner locked. Only a confirmed ENOENT registration or a confirmed missing or replaced lease + * file ends it; any other error (permissions, an unmounted root mid-read) keeps the owner running. */ -const registrationConfirmedGone = Effect.fn("StackHost.registrationConfirmedGone")(function* ( +const ownershipEnded = Effect.fn("StackHost.ownershipEnded")(function* ( state: StackNamespace.Interface, id: string, + pathHeld: Effect.Effect, ) { - const saved = yield* state.read(id).pipe( + return yield* Effect.all([state.read(id), pathHeld]).pipe( + Effect.map(([saved, held]) => saved === undefined || !held), Effect.tapError((cause) => - Effect.logWarning( - "Could not confirm the stack registration; keeping the owner running", - cause, - ), + Effect.logWarning("Could not confirm the stack ownership; keeping the owner running", cause), ), - Effect.option, + Effect.orElseSucceed(() => false), ); - return Option.isSome(saved) && saved.value === undefined; }); -/** - * Polls for a confirmed-gone registration and offers `registration-gone` once. - */ -const pollRegistration = Effect.fn("StackHost.pollRegistration")(function* ( +/** Polls ownership and offers `ownership-ended` once it ends. */ +const pollOwnership = Effect.fn("StackHost.pollOwnership")(function* ( state: StackNamespace.Interface, id: string, + pathHeld: Effect.Effect, events: Queue.Queue, ) { const interval = yield* RegistrationCheckInterval; while (true) { yield* Effect.sleep(interval); - if (yield* registrationConfirmedGone(state, id)) { - yield* Queue.offer(events, "registration-gone"); + if (yield* ownershipEnded(state, id, pathHeld)) { + yield* Queue.offer(events, "ownership-ended"); return; } } @@ -570,7 +550,7 @@ export const runStackHost = Effect.fn("StackHost.run")( yield* Effect.forkScoped( creatorGone.pipe(Effect.andThen(Queue.offer(events, "creator-gone"))), ); - yield* Effect.forkScoped(pollRegistration(state, id, events)); + yield* Effect.forkScoped(pollOwnership(state, id, lease.pathHeld, events)); yield* options.onReady?.(access) ?? Effect.void; yield* Effect.forkScoped( sweepOrphans({ @@ -611,8 +591,8 @@ export const runStackHost = Effect.fn("StackHost.run")( ); return; } - if (event === "registration-gone") { - yield* started.abandon; + if (event === "ownership-ended") { + yield* started.release; return; } if (event !== "done") @@ -620,10 +600,6 @@ export const runStackHost = Effect.fn("StackHost.run")( Effect.tapCause((cause) => Effect.logError("Stack shutdown failed", cause)), Effect.ignore, ); - // A registration deletion that arrives while a data-preserving stop is already running - // only reaches `events` as a queued, now-unread `registration-gone`: a direct re-read - // through the same state API the poll uses catches it before the owner exits. - if (yield* registrationConfirmedGone(state, id)) yield* started.abandon; }), ).pipe( Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp)), diff --git a/packages/stack/src/Sweep.integration.test.ts b/packages/stack/src/Sweep.integration.test.ts index e36b3c572d..8bf25aebde 100644 --- a/packages/stack/src/Sweep.integration.test.ts +++ b/packages/stack/src/Sweep.integration.test.ts @@ -19,7 +19,7 @@ import { watch as nodeWatch } from "node:fs"; import { launchHost } from "./HostProcess.ts"; import { makeContainerRuntime, type EngineTarget } from "./runtime/Container.ts"; import * as StackNamespace from "./StackNamespace.ts"; -import { reclaimDeletedStack, sweepOrphans } from "./Sweep.ts"; +import { reclaimDeletedStack, reclaimStack, sweepOrphans } from "./Sweep.ts"; import { engineStub, makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; import { awaitContainerRemoved } from "../tests/engine-events.ts"; import { engineTarget, testEngine } from "../tests/engine-target.ts"; @@ -266,6 +266,38 @@ it.live( ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); +it.live("reclaiming a session stack refuses a symlinked data directory and keeps its target", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-sweep-symlinked-data-" }); + const stateRoot = path.join(root, "state"); + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root: stateRoot })), + StackNamespace.Service, + ); + const stack = saved("symlinked-data", root, "native", "session"); + yield* state.save(stack); + const outside = path.join(root, "outside"); + yield* fs.makeDirectory(outside); + yield* fs.writeFileString(path.join(outside, "precious.txt"), "keep"); + yield* fs.symlink(outside, path.join(stateRoot, stack.id, "data")); + + const failure = yield* reclaimStack({ + state, + stateRoot, + cacheRoot: path.join(root, "cache"), + id: stack.id, + }).pipe(Effect.flip); + + expect(failure.message).toContain("symlink"); + expect(yield* fs.readFileString(path.join(outside, "precious.txt"))).toBe("keep"); + expect(yield* state.read(stack.id), "the registration stays for a retry").toBeDefined(); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + it.live("sweeps deleted stacks of its own root only, on the engine its owner pinned", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/Sweep.ts b/packages/stack/src/Sweep.ts index 912b3ea58a..d46f25183d 100644 --- a/packages/stack/src/Sweep.ts +++ b/packages/stack/src/Sweep.ts @@ -64,7 +64,7 @@ export const reclaimStack = Effect.fn("Sweep.reclaimStack")(function* (options: if (!(yield* holdLease(state, id))) return false; const saved = yield* state.read(id); if (saved === undefined) return true; - const dataRoot = yield* StackNamespace.resolveStackDataRoot(options.stateRoot, id); + const dataRoot = yield* StackNamespace.stackDataRootLabel(options.stateRoot, id); if (saved.lifetime === "session") yield* destroyStack(state, saved, dataRoot, options.cacheRoot); else { diff --git a/packages/stack/src/namespace/Lease.ts b/packages/stack/src/namespace/Lease.ts index b7d55ced53..411192a9c2 100644 --- a/packages/stack/src/namespace/Lease.ts +++ b/packages/stack/src/namespace/Lease.ts @@ -40,7 +40,10 @@ interface LeaseHandle { readonly stackId: string; readonly ownerLog: string; readonly publishHolder: (record: LeaseHolder) => Effect.Effect; + /** Removes the holder record while the lease path still names the file this lease locked. */ readonly retractHolder: Effect.Effect; + /** False once the lease path no longer names the file this lease locked (confirmed missing or replaced). */ + readonly pathHeld: Effect.Effect; } export interface Interface { @@ -85,11 +88,10 @@ export const make = ( /** An open file that was unlinked: SQLite IOERR_VNODE on macOS, IOERR_FSTAT on Linux. */ const isMoved = (error: NamespaceError) => errcode(error.cause, 6922) || errcode(error.cause, 1802); + const identityOf = (info: FileSystem.File.Info) => + `${info.dev}:${Option.getOrElse(info.ino, () => "")}`; const fileIdentity = (file: string) => - fs.stat(file).pipe( - Effect.map((info) => `${info.dev}:${Option.getOrElse(info.ino, () => "")}`), - Effect.option, - ); + fs.stat(file).pipe(Effect.map(identityOf), Effect.option); /** * Deletes an unregistered stack's lease file while it is still the file locked as `identity`; * `false` means it is still in place. @@ -198,9 +200,18 @@ export const make = ( }), ), ); + const pathHeld = fs.stat(target).pipe( + Effect.map((info) => identityOf(info) === held), + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed(false), + ), + Effect.mapError((cause) => namespaceError("lease", cause)), + ); return { stackId: id, ownerLog: outcome.ownerLog, + pathHeld, publishHolder: Effect.fn("Namespace.Lease.publishHolder")(function* (record: LeaseHolder) { const serialized = yield* Schema.encodeEffect(Schema.fromJsonString(LeaseHolder))( record, @@ -214,8 +225,14 @@ export const make = ( platform: options.platform, }); }), - retractHolder: fs.remove(ownerPath(id), { force: true }).pipe( - Effect.mapError((cause) => namespaceError("remove", cause)), + retractHolder: pathHeld.pipe( + Effect.flatMap((held) => + held + ? fs + .remove(ownerPath(id), { force: true }) + .pipe(Effect.mapError((cause) => namespaceError("remove", cause))) + : Effect.void, + ), Effect.withSpan("Namespace.Lease.retractHolder"), ), } satisfies LeaseHandle; diff --git a/packages/stack/src/namespace/Namespace.integration.test.ts b/packages/stack/src/namespace/Namespace.integration.test.ts index 3d8a29acd3..645663b1e6 100644 --- a/packages/stack/src/namespace/Namespace.integration.test.ts +++ b/packages/stack/src/namespace/Namespace.integration.test.ts @@ -696,6 +696,39 @@ describe("stack owner lease", () => { ), ); + // Windows refuses to delete a stack directory while its lease file is open. + it.live.skipIf(process.platform === "win32")( + "keeps a successor's discovery record when the holder of a removed stack retracts its own", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lease-record-" }); + const state = yield* makeTestState(root); + const record = (pid: number): StackNamespace.LeaseHolder => ({ + role: "sweeper", + pid, + startedAt: "2026-01-01T00:00:00.000Z", + }); + const first = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(first, Exit.void)); + const removed = yield* state.acquireLease("gone").pipe(Scope.provide(first)); + yield* removed.publishHolder(record(1)); + yield* fs.remove(`${root}/gone`, { recursive: true }); + + const successor = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(successor, Exit.void)); + const current = yield* state.acquireLease("gone").pipe(Scope.provide(successor)); + yield* current.publishHolder(record(2)); + yield* removed.retractHolder; + + expect(yield* state.readHolder("gone")).toEqual(record(2)); + yield* current.retractHolder; + expect(yield* state.readHolder("gone")).toBeUndefined(); + }), + ), + ); + it.live("reports a free lease without creating a lease file", () => run( Effect.gen(function* () { diff --git a/packages/stack/src/namespace/Paths.integration.test.ts b/packages/stack/src/namespace/Paths.integration.test.ts index 5e5ed5cf61..d134d9c84b 100644 --- a/packages/stack/src/namespace/Paths.integration.test.ts +++ b/packages/stack/src/namespace/Paths.integration.test.ts @@ -1,7 +1,7 @@ import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Data, Effect, FileSystem, Path } from "effect"; -import { borrow, destroyOwnedRoot } from "./Paths.ts"; +import { borrow, destroyOwnedRoot, resolveStackDataRoot } from "./Paths.ts"; class TestPathsError extends Data.TaggedError("TestPathsError")<{ readonly operation: string; @@ -221,3 +221,25 @@ describe("ownership is by location", () => { ), ); }); + +describe("the stack data root", () => { + it.live("is refused at launch when it is a symlink", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-data-root-symlink-" }); + const outside = path.join(root, "outside"); + yield* fs.makeDirectory(outside); + yield* fs.makeDirectory(path.join(root, "state", "stack"), { recursive: true }); + yield* fs.symlink(outside, path.join(root, "state", "stack", "data")); + + const failure = yield* resolveStackDataRoot(path.join(root, "state"), "stack").pipe( + Effect.flip, + ); + + expect(failure.message).toContain("symlink"); + }), + ), + ); +}); diff --git a/packages/stack/src/namespace/Paths.ts b/packages/stack/src/namespace/Paths.ts index 024d128dc0..948e197119 100644 --- a/packages/stack/src/namespace/Paths.ts +++ b/packages/stack/src/namespace/Paths.ts @@ -1,4 +1,5 @@ import { Effect, FileSystem, Path } from "effect"; +import { namespaceError } from "./Capabilities.ts"; import { lstatPath } from "./drivers/FileSystem.ts"; const BorrowedPathBrand: unique symbol = Symbol("Namespace.BorrowedPath"); @@ -66,23 +67,6 @@ export const borrow = Effect.fn("Namespace.Paths.borrow")( }), ); -/** - * The canonical data root of a stack: its `data` directory under `stateRoot`, created if missing - * and resolved through symlinks. Launch and every cleanup path derive container labels and native - * socket directories from this one value, so each reaches the same resources however the state - * root is spelled. - */ -export const resolveStackDataRoot = Effect.fn("Namespace.Paths.resolveStackDataRoot")(function* ( - stateRoot: string, - stackId: string, -) { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const dataRoot = path.join(stateRoot, stackId, "data"); - yield* fs.makeDirectory(dataRoot, { recursive: true }); - return yield* fs.realPath(dataRoot); -}); - /** * The data-root label its containers carry, derived without creating the directory, for a stack * whose directory may be gone; it equals {@link resolveStackDataRoot} while the directory exists. @@ -177,3 +161,40 @@ export const destroyOwnedRoot = Effect.fn("Namespace.Paths.destroyOwnedRoot")( .pipe(Effect.mapError((cause) => onError("destroy", cause))); }), ); + +/** + * Refuses a stack's `data` directory when it is a symlink or resolves outside the stack + * directory, so a destroy never empties a target the stack does not own. A missing directory + * passes: only launch creates it. + */ +export const confirmStackDataRoot = Effect.fn("Namespace.Paths.confirmStackDataRoot")(function* ( + stateRoot: string, + stackId: string, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stackRoot = path.join(stateRoot, stackId); + const dataRoot = path.join(stackRoot, "data"); + const present = yield* fs + .exists(dataRoot) + .pipe(Effect.mapError((cause) => namespaceError("destroy", cause))); + if (present) yield* confirmRealOwnedDirectory(fs, path, dataRoot, stackRoot, namespaceError); +}); + +/** + * The canonical data root of a stack: its `data` directory under `stateRoot`, created if missing + * and resolved through symlinks. Launch and every cleanup path derive container labels and native + * socket directories from this one value, so each reaches the same resources however the state + * root is spelled. A `data` directory that is itself a symlink is refused. + */ +export const resolveStackDataRoot = Effect.fn("Namespace.Paths.resolveStackDataRoot")(function* ( + stateRoot: string, + stackId: string, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dataRoot = path.join(stateRoot, stackId, "data"); + yield* fs.makeDirectory(dataRoot, { recursive: true }); + yield* confirmStackDataRoot(stateRoot, stackId); + return yield* fs.realPath(dataRoot); +}); From 681254758d82930a59961ff90358efe147a62d1d Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 14:26:21 +0200 Subject: [PATCH 125/142] test(stack): assert that lost ownership stops containers and keeps data --- ...ost.container-shutdown.integration.test.ts | 193 +++++------------- .../stack/src/lifetime.integration.test.ts | 14 +- .../tests/short-registration-poll-fixture.ts | 2 +- 3 files changed, 59 insertions(+), 150 deletions(-) diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index 028800918a..7825d523e1 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -113,7 +113,7 @@ const startHost = (stateRoot: string, cacheRoot: string, stackId: string, projec }); it.live.skipIf(process.platform === "win32")( - "stops, destroys, and abandons only containers owned by the same stack id and data root", + "stops, destroys, and releases only containers owned by the same stack id and data root", () => Effect.scoped( Effect.gen(function* () { @@ -248,8 +248,8 @@ it.live.skipIf(process.platform === "win32")( expect(yield* containers(stackId, dataB), "destroy removes B containers").toEqual([]); expect(yield* (yield* stateFor(rootB)).read(stackId)).toBeUndefined(); - // Abandonment phase: two fresh owners share this id again, each under its own root. - // Abandoning A must only ever reach containers carrying A's own `stack-root` label. + // Registration-loss phase: two fresh owners share this id again, each under its own root. + // Ending A's ownership must only ever reach containers carrying A's own `stack-root` label. const stateA3 = yield* stateFor(rootA); const stateB3 = yield* stateFor(rootB); yield* stateA3.save({ @@ -323,13 +323,13 @@ it.live.skipIf(process.platform === "win32")( expect( yield* containers(stackId, dataA), - "abandoning A removes only A's containers", + "ending A's ownership stops only A's containers", ).toEqual([]); expect( yield* containers(stackId, dataB), - "abandoning A leaves B's containers running", + "ending A's ownership leaves B's containers running", ).toEqual(idsB3); - // B's own container still answers a real readiness probe, proving abandonment never + // B's own container still answers a real readiness probe, proving A's owner never // touched it. yield* clientB3.readyService({ id: mailB3.id }); @@ -346,18 +346,19 @@ it.live.skipIf(process.platform === "win32")( ); it.live.skipIf(process.platform === "win32")( - "removes its containers and exits when its registration is confirmed gone", + "stops its containers, keeps its data and exits when its registration is confirmed gone", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; - const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-abandon-docker-" }); - const stackId = `abandon-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; - const dataRoot = yield* makeDockerDatabaseRoot("stack-abandon-docker-data-", stackId).pipe( - Effect.flatMap(fs.realPath), - ); + const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-registration-lost-" }); + const stackId = `registration-lost-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const dataRoot = yield* makeDockerDatabaseRoot( + "stack-registration-lost-data-", + stackId, + ).pipe(Effect.flatMap(fs.realPath)); const stateRoot = path.dirname(path.dirname(dataRoot)); const cacheRoot = `${base}/cache`; const state = yield* stateFor(stateRoot); @@ -366,7 +367,7 @@ it.live.skipIf(process.platform === "win32")( runtime: testEngine, identity: { projectRoot: `${base}/project`, - branchContext: "abandon-docker-test", + branchContext: "registration-lost-test", stackName: stackId, }, instances: [], @@ -387,8 +388,8 @@ it.live.skipIf(process.platform === "win32")( service: "database", config: { version: "17", - databasePassword: Redacted.make("abandon-docker-password"), - jwtSecret: Redacted.make("abandon-docker-jwt-secret-at-least-thirty-two-characters"), + databasePassword: Redacted.make("registration-lost-password"), + jwtSecret: Redacted.make("registration-lost-jwt-secret-at-least-thirty-two-characters"), jwtExpiry: 3600, }, endpoints: { sql: { port: "auto" } }, @@ -414,15 +415,17 @@ it.live.skipIf(process.platform === "win32")( Effect.timeout("30 seconds"), ); - expect(yield* containers(stackId, dataRoot), "abandonment removes containers").toEqual([]); + expect(yield* containers(stackId, dataRoot), "the owner stopped its containers").toEqual( + [], + ); expect( yield* fs.exists(`${stateRoot}/${stackId}/state.json`), "no registration is republished", ).toBe(false); expect( yield* fs.exists(containerEnvRoot), - "abandonment removes the shared container-env scratch directory", - ).toBe(false); + "the data root is left in place for a successor or `destroy`", + ).toBe(true); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 180_000 }, @@ -480,17 +483,18 @@ const triggerStop = (trigger: StopTrigger, access: HostAccess) => ? Effect.sync(() => process.kill(access.endpoint.pid, "SIGTERM")) : Effect.forkScoped(shutdownOwner(access, false).pipe(Effect.ignore)); -const abandonsWhileStopRuns = (trigger: StopTrigger) => +const losesRegistrationWhileStopRuns = (trigger: StopTrigger) => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; - const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-abandon-gated-" }); - const stackId = `abandon-gated-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; - const dataRoot = yield* makeDockerDatabaseRoot("stack-abandon-gated-data-", stackId).pipe( - Effect.flatMap(fs.realPath), - ); + const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-registration-lost-gated-" }); + const stackId = `registration-lost-gated-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const dataRoot = yield* makeDockerDatabaseRoot( + "stack-registration-lost-gated-data-", + stackId, + ).pipe(Effect.flatMap(fs.realPath)); const stateRoot = path.dirname(path.dirname(dataRoot)); const cacheRoot = `${base}/cache`; const gateDir = `${base}/gate`; @@ -501,7 +505,7 @@ const abandonsWhileStopRuns = (trigger: StopTrigger) => runtime: testEngine, identity: { projectRoot: `${base}/project`, - branchContext: "abandon-gated-test", + branchContext: "registration-lost-gated-test", stackName: stackId, }, instances: [], @@ -533,8 +537,10 @@ const abandonsWhileStopRuns = (trigger: StopTrigger) => service: "database", config: { version: "17", - databasePassword: Redacted.make("abandon-gated-password"), - jwtSecret: Redacted.make("abandon-gated-jwt-secret-at-least-thirty-two-characters"), + databasePassword: Redacted.make("registration-lost-gated-password"), + jwtSecret: Redacted.make( + "registration-lost-gated-jwt-secret-at-least-thirty-two-characters", + ), jwtExpiry: 3600, }, endpoints: { sql: { port: "auto" } }, @@ -571,115 +577,37 @@ const abandonsWhileStopRuns = (trigger: StopTrigger) => ).toBe(false); expect( yield* fs.exists(containerEnvRoot), - "the registration lost during the gated stop still reaches abandonment's cleanup", - ).toBe(false); - expect(yield* containers(stackId, dataRoot), "no containers are left behind").toEqual([]); + "the stop that was running when the registration disappeared leaves the data root in place", + ).toBe(true); + expect(yield* containers(stackId, dataRoot), "the held stop still completed").toEqual([]); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))); it.live.skipIf(process.platform === "win32")( - "abandons a stack whose registration disappears while a signal-driven stop is still running", - () => abandonsWhileStopRuns("signal"), + "finishes a signal-driven stop and exits when the registration disappears mid-stop", + () => losesRegistrationWhileStopRuns("signal"), { timeout: 180_000 }, ); it.live.skipIf(process.platform === "win32")( - "abandons a stack whose registration disappears while an HTTP-driven stop is still running", - () => abandonsWhileStopRuns("http"), + "finishes an HTTP-driven stop and exits when the registration disappears mid-stop", + () => losesRegistrationWhileStopRuns("http"), { timeout: 180_000 }, ); it.live.skipIf(process.platform === "win32")( - "removes an orphaned container by label alone, independent of any instance's own cleanup", + "stops its containers and exits when its whole state root is confirmed gone", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; - const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-abandon-docker-label-" }); - const stackId = `abandon-label-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; - const dataRoot = yield* makeDockerDatabaseRoot( - "stack-abandon-docker-label-data-", - stackId, - ).pipe(Effect.flatMap(fs.realPath)); - const stateRoot = path.dirname(path.dirname(dataRoot)); - const cacheRoot = `${base}/cache`; - const state = yield* stateFor(stateRoot); - yield* state.save({ - id: stackId, - runtime: testEngine, - identity: { - projectRoot: `${base}/project`, - branchContext: "abandon-docker-label-test", - stackName: stackId, - }, - instances: [], - lifetime: "detached" as const, - composition: { members: [], dependencies: [] }, - }); - yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); - const access = yield* launchHost(state, { - stateRoot, - cacheRoot, - stackId, - entrypoint: shortRegistrationPollFixture, - }); - // No registered service at all, so the per-instance cleanup loop has nothing to do: this - // container carries only the stack's identity label, simulating a leaked storage helper - // no claim or helper-registry bookkeeping ever reaches. Only the - // registration-independent label sweep can remove it. - yield* engine([ - "run", - "-d", - "--name", - `supabase-orphan-${stackId}`, - "--label", - `com.supabase.stack=${stackId}`, - "--label", - `com.supabase.stack-root=${dataRoot}`, - "--label", - "com.supabase.stack-managed=true", - helperImage, - "/bin/sh", - "-c", - "trap : TERM INT; while :; do sleep 3600; done", - ]); - expect( - (yield* containers(stackId, dataRoot)).length, - "the orphan container exists before abandonment", - ).toBeGreaterThan(0); - - const leaseReleased = yield* watchLeaseRelease(stateRoot, stackId); - yield* fs.remove(`${stateRoot}/${stackId}/state.json`); - yield* leaseReleased; - yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( - Effect.timeout("30 seconds"), + const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-root-lost-" }); + const stackId = `state-root-lost-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const dataRoot = yield* makeDockerDatabaseRoot("stack-state-root-lost-data-", stackId).pipe( + Effect.flatMap(fs.realPath), ); - - expect( - yield* containers(stackId, dataRoot), - "abandonment removes the orphan container by label alone", - ).toEqual([]); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), - { timeout: 180_000 }, -); - -it.live.skipIf(process.platform === "win32")( - "removes its containers and exits when its whole state root is confirmed gone", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-abandon-docker-root-" }); - const stackId = `abandon-root-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; - const dataRoot = yield* makeDockerDatabaseRoot( - "stack-abandon-docker-root-data-", - stackId, - ).pipe(Effect.flatMap(fs.realPath)); const stateRoot = path.dirname(path.dirname(dataRoot)); const cacheRoot = `${base}/cache`; const state = yield* stateFor(stateRoot); @@ -688,7 +616,7 @@ it.live.skipIf(process.platform === "win32")( runtime: testEngine, identity: { projectRoot: `${base}/project`, - branchContext: "abandon-docker-root-test", + branchContext: "state-root-lost-test", stackName: stackId, }, instances: [], @@ -721,28 +649,9 @@ it.live.skipIf(process.platform === "win32")( const status = yield* client.status({ id: mail.id }); const port = status.endpoints.find((endpoint) => endpoint.name === "http")?.port; if (port === undefined) return yield* Effect.die("Missing mail http endpoint"); - // Reachable only by the final label sweep, which runs only if confirming the absent - // registration does not fail on the vanished registry lock. - yield* engine([ - "run", - "-d", - "--name", - `supabase-orphan-${stackId}`, - "--label", - `com.supabase.stack=${stackId}`, - "--label", - `com.supabase.stack-root=${dataRoot}`, - "--label", - "com.supabase.stack-managed=true", - helperImage, - "/bin/sh", - "-c", - "trap : TERM INT; while :; do sleep 3600; done", - ]); - // Deletes the actual `` itself, not just the entries inside // `/`: its sibling `.registry-lock.sqlite` (`Ports.ts`'s per-root - // reservation registry) is gone too, so cleanup can only come from the owner's in-memory + // reservation registry) is gone too, so the stop can only come from the owner's in-memory // resources, and must never open that registry at all. The lease file goes with it, so // this relies on budgeted polling rather than a lease-release subscription. yield* fs.remove(stateRoot, { recursive: true, force: true }); @@ -757,18 +666,18 @@ it.live.skipIf(process.platform === "win32")( expect( yield* containers(stackId, dataRoot), - "abandonment removes the orphan and the service containers with the registry gone", + "the owner stops its service containers with the registry gone", ).toEqual([]); // The vanished registry cannot release the port reservation: a fresh stack can still // claim the exact same port, through `Ports.ts`'s own lazy reclamation once the former // holder's registration is confirmed gone. const reclaimedBase = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-abandon-docker-root-reclaim-", + prefix: "stack-state-root-lost-reclaim-", }); - const reclaimedStackId = `abandon-root-reclaim-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const reclaimedStackId = `state-root-lost-reclaim-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; const reclaimedDataRoot = yield* makeDockerDatabaseRoot( - "stack-abandon-docker-root-reclaim-data-", + "stack-state-root-lost-reclaim-data-", reclaimedStackId, ).pipe(Effect.flatMap(fs.realPath)); const reclaimedStateRoot = path.dirname(path.dirname(reclaimedDataRoot)); diff --git a/packages/stack/src/lifetime.integration.test.ts b/packages/stack/src/lifetime.integration.test.ts index 654d26e68a..fb73e3afdf 100644 --- a/packages/stack/src/lifetime.integration.test.ts +++ b/packages/stack/src/lifetime.integration.test.ts @@ -204,14 +204,14 @@ it.live("lets only the creating handle start a session stack's owner", () => ); it.live.skipIf(process.platform === "win32")( - "exits, stops its native workload and releases its port when its registration is confirmed gone", + "exits and stops its native workload, freeing its port, when its registration is confirmed gone", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-abandon-native-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-registration-lost-native-" }); const stateRoot = `${root}/state`; const state = yield* stateFor(stateRoot); - const stackId = "abandon-native"; + const stackId = "registration-lost-native"; yield* state.save({ id: stackId, runtime: "native", @@ -253,19 +253,19 @@ it.live.skipIf(process.platform === "win32")( // this final, now-fast confirmation. yield* waitForOwnerExit(ownerPid, ownerExitProbe(fs)).pipe(Effect.timeout("10 seconds")); - expect(owned.filter(alive), "native processes die with their abandoned owner").toEqual([]); + expect(owned.filter(alive), "native processes die with their owner").toEqual([]); expect( yield* fs.exists(`${stateRoot}/${stackId}/state.json`), "no registration is republished", ).toBe(false); - // The port reservation is released: a fresh stack can claim the exact same port. + // The stopped workload frees its port: a fresh stack can claim the exact same port. const reclaimedRoot = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-abandon-native-reclaim-", + prefix: "stack-registration-lost-native-reclaim-", }); const reclaimedStateRoot = `${reclaimedRoot}/state`; const reclaimedState = yield* stateFor(reclaimedStateRoot); - const reclaimedId = "abandon-native-reclaim"; + const reclaimedId = "registration-lost-native-reclaim"; yield* reclaimedState.save({ id: reclaimedId, runtime: "native", diff --git a/packages/stack/tests/short-registration-poll-fixture.ts b/packages/stack/tests/short-registration-poll-fixture.ts index 052241dd31..32d3226646 100644 --- a/packages/stack/tests/short-registration-poll-fixture.ts +++ b/packages/stack/tests/short-registration-poll-fixture.ts @@ -1,5 +1,5 @@ import { runHostProcess } from "../src/internal/host-process.ts"; -/** An owner whose registration-loss poll runs every 200ms, so abandonment tests need not wait the production interval. */ +/** An owner whose registration-loss poll runs every 200ms, so ownership-loss tests need not wait the production interval. */ if (import.meta.main) await runHostProcess(process.argv.slice(2), { registrationCheckInterval: "200 millis" }); From c7293408fad239ad9902f82e984748f183ac42bf Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 14:29:05 +0200 Subject: [PATCH 126/142] fix(stack): serialize credential refresh with service starts --- packages/stack/src/Orchestrator.ts | 31 ++++++++++++ packages/stack/src/Owner.ts | 75 +++++++++++++----------------- 2 files changed, 63 insertions(+), 43 deletions(-) diff --git a/packages/stack/src/Orchestrator.ts b/packages/stack/src/Orchestrator.ts index 92f04c5d24..dce4e4ad4f 100644 --- a/packages/stack/src/Orchestrator.ts +++ b/packages/stack/src/Orchestrator.ts @@ -182,6 +182,16 @@ export interface Interface, ) => Effect.Effect; + /** + * Runs `effect` under the lifecycle gate once every listed instance is stopped with wake + * disabled, failing with `refuse` for the first that is not. Callers take the cross-process + * state lock first. + */ + readonly whileStopped: ( + ids: Iterable, + refuse: (id: string) => F, + effect: Effect.Effect, + ) => Effect.Effect; readonly startComposition: Effect.Effect, OrchestratorError | ServiceError>; readonly stopComposition: Effect.Effect, OrchestratorError | ServiceError>; readonly restartComposition: Effect.Effect< @@ -1209,6 +1219,27 @@ export const make = Effect.fn("Orchestrator.make")(function* ( + ids: Iterable, + refuse: (id: string) => F, + effect: Effect.Effect, + ) => + withGate( + Effect.gen(function* () { + const state = yield* SubscriptionRef.get(lifecycle); + for (const id of ids) { + yield* node(id); + if ( + !isStoppedAndWakeDisabled({ + lifecycle: lifecycleOf(state, id), + wakeEnabled: wakeEnabledOf(state, id), + }) + ) + return yield* Effect.fail(refuse(id)); + } + return yield* effect; + }), + ).pipe(Effect.withSpan("Orchestrator.whileStopped")), start, ready, stop, diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index 95dadc6ff3..95f413d60c 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -350,30 +350,6 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ), ); - const requireStopped = (current: SavedStack) => - Effect.forEach( - new Set([ - ...current.composition.members.map(({ id }) => id), - ...current.composition.dependencies.flatMap(({ from, to }) => [from, to]), - ...current.instances - .filter(({ creation }) => consumesCredentials(creation)) - .map(({ id }) => id), - ]), - (id) => - orchestrator.status(id).pipe( - Effect.flatMap((status) => - Orchestrator.isStoppedAndWakeDisabled(status) - ? Effect.void - : Effect.fail( - new CredentialError({ - message: `Service ${id} must be stopped with wake disabled before stack credentials change`, - }), - ), - ), - ), - { discard: true }, - ); - const resolveStackCredentials = Effect.fn("Owner.resolveStackCredentials")(function* ( overrides: Effect.Success>, keys?: StackKeysInput, @@ -388,29 +364,42 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { yield* options.state.save({ ...current, credentials: next }); return next; } - yield* requireStopped(current); + const consumers = current.instances.filter(({ creation }) => consumesCredentials(creation)); const refreshed = new Map( - yield* Effect.forEach( - current.instances.filter(({ creation }) => consumesCredentials(creation)), - ({ id, creation }) => - refreshCredentials(creation, previous, next).pipe( - Effect.map((refreshedCreation) => [id, refreshedCreation] as const), - ), + yield* Effect.forEach(consumers, ({ id, creation }) => + refreshCredentials(creation, previous, next).pipe( + Effect.map((refreshedCreation) => [id, refreshedCreation] as const), + ), ), ); - yield* options.state.save({ - ...current, - credentials: next, - instances: current.instances.map((instance) => { - const creation = refreshed.get(instance.id); - return creation === undefined ? instance : { ...instance, creation }; + yield* orchestrator.whileStopped( + new Set([ + ...current.composition.members.map(({ id }) => id), + ...current.composition.dependencies.flatMap(({ from, to }) => [from, to]), + ...consumers.map(({ id }) => id), + ]), + (id) => + new CredentialError({ + message: `Service ${id} must be stopped with wake disabled before stack credentials change`, + }), + Effect.gen(function* () { + yield* options.state.save({ + ...current, + credentials: next, + instances: current.instances.map((instance) => { + const creation = refreshed.get(instance.id); + return creation === undefined ? instance : { ...instance, creation }; + }), + }); + yield* Effect.forEach( + refreshed, + ([id, creation]) => + orchestrator + .get(id) + .pipe(Effect.flatMap((entry) => Ref.set(entry.creation, creation))), + { discard: true }, + ); }), - }); - yield* Effect.forEach( - refreshed, - ([id, creation]) => - orchestrator.get(id).pipe(Effect.flatMap((entry) => Ref.set(entry.creation, creation))), - { discard: true }, ); yield* Effect.annotateCurrentSpan("refreshed_instances", refreshed.size); return next; From 0cf63317e9f9329b462d28a320e8b465f91275e1 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 14:30:34 +0200 Subject: [PATCH 127/142] fix(stack): fail destroy when the container engine is unreachable and explain unreadable stacks --- apps/cli/src/command-internal/stack-api.ts | 12 --- .../stack-shadow.integration.test.ts | 19 +++-- apps/cli/src/command-internal/stack-shadow.ts | 10 +-- .../stack/destroy/SIDE_EFFECTS.md | 27 +++--- .../stack/destroy/destroy.handler.ts | 48 +++++------ .../stack/destroy/destroy.integration.test.ts | 19 +++++ ...oy.runtime-unavailable.integration.test.ts | 84 +++++++------------ .../stack/stack.shared.integration.test.ts | 32 +++++++ .../experimental/stack/stack.shared.ts | 16 +--- ...oy.runtime-unavailable.integration.test.ts | 18 ++-- packages/stack/src/effect.integration.test.ts | 29 +++++++ packages/stack/src/effect.ts | 61 ++++---------- 12 files changed, 186 insertions(+), 189 deletions(-) diff --git a/apps/cli/src/command-internal/stack-api.ts b/apps/cli/src/command-internal/stack-api.ts index 019ea02139..0aa1c8d538 100644 --- a/apps/cli/src/command-internal/stack-api.ts +++ b/apps/cli/src/command-internal/stack-api.ts @@ -1,4 +1,3 @@ -import type { ContainerEngine } from "@supabase/stack/internal/engine"; import { Context, Crypto, Effect, FileSystem, Layer, Option, Path, Scope } from "effect"; import { FetchHttpClient, HttpClient } from "effect/unstable/http"; import { ChildProcessSpawner } from "effect/unstable/process"; @@ -16,7 +15,6 @@ import { type Stack, type StackError, } from "@supabase/stack/effect"; -import { containerEngineName } from "./stack-runtime.ts"; type DiscoverResult = Effect.Success>; @@ -77,14 +75,4 @@ export const stackApiLayer = Layer.effect( }), ).pipe(Layer.provide(FetchHttpClient.layer)); -/** Describes why a destroy left engine resources behind, without promising automatic cleanup. */ -export const skippedRuntimeCleanupWarning = ( - subject: string, - stackId: string, - engine: ContainerEngine, -): string => { - const name = containerEngineName(engine); - return `${name} was unavailable, so ${name} resources for ${subject} were not removed. Restore ${name} and run "supabase stack destroy --stack-id ${stackId}" again to finish removing it.`; -}; - export type { Stack }; diff --git a/apps/cli/src/command-internal/stack-shadow.integration.test.ts b/apps/cli/src/command-internal/stack-shadow.integration.test.ts index d58c6e5708..b3e4ba76b1 100644 --- a/apps/cli/src/command-internal/stack-shadow.integration.test.ts +++ b/apps/cli/src/command-internal/stack-shadow.integration.test.ts @@ -458,14 +458,14 @@ describe("stack shadow databases", () => { ); it.live( - "warns that a shadow's engine cleanup is pending when destroy skips its engine", + "reports a shadow whose destroy fails because its engine is unreachable", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-shadow-skipped-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-shadow-unreachable-" }); const output = mockOutput(); - // A real registration whose database creation fails fast and whose destroy reports skipped cleanup. + // A real registration whose database creation fails fast and whose destroy fails with the engine unreachable. const api = Layer.effect( StackApi, Effect.gen(function* () { @@ -481,10 +481,13 @@ describe("stack shadow databases", () => { create: () => Effect.fail(new StackError({ operation: "create", message: "injected" })), }, - destroy: Effect.succeed({ - runtimeCleanup: "skipped", - engine: "docker", - } as const), + destroy: Effect.fail( + new StackError({ + operation: "destroy", + message: "Docker CLI or daemon isn't reachable", + reason: "runtime-unavailable", + }), + ), })), ), }); @@ -508,7 +511,7 @@ describe("stack shadow databases", () => { ); expect(output.stderrText).toMatch( - /Warning: Docker was unavailable, so Docker resources for shadow stack [0-9a-f]{64} were not removed\. Restore Docker and run "supabase stack destroy --stack-id [0-9a-f]{64}" again to finish removing it\.\n/u, + /Failed to destroy shadow stack [0-9a-f]{64}: Docker CLI or daemon isn't reachable\.\n/u, ); }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), 120_000, diff --git a/apps/cli/src/command-internal/stack-shadow.ts b/apps/cli/src/command-internal/stack-shadow.ts index 729d5a77cf..764bc4eb93 100644 --- a/apps/cli/src/command-internal/stack-shadow.ts +++ b/apps/cli/src/command-internal/stack-shadow.ts @@ -3,7 +3,7 @@ import type { DatabaseInstance, Stack } from "@supabase/stack/effect"; import { CommandSettings } from "../config/command-settings.service.ts"; import { Output } from "../shared/output/output.service.ts"; import { RuntimeInfo } from "../shared/runtime/runtime-info.service.ts"; -import { skippedRuntimeCleanupWarning, StackApi } from "./stack-api.ts"; +import { StackApi } from "./stack-api.ts"; import { initializeStackDatabase } from "./stack-bootstrap.ts"; import { stackProjectRuntime } from "./stack-local-database.ts"; import { selectStackRuntime, type StackRuntime } from "./stack-runtime.ts"; @@ -233,14 +233,6 @@ export const stackAcquireShadowDatabase = Effect.fn("StackShadow.acquire")(funct const output = yield* Output; const namespace = yield* Effect.acquireRelease(acquireNamespace(opts), ({ stack }) => stack.destroy.pipe( - Effect.flatMap((result) => - result.runtimeCleanup === "skipped" - ? output.raw( - `Warning: ${skippedRuntimeCleanupWarning(`shadow stack ${stack.id}`, stack.id, result.engine)}\n`, - "stderr", - ) - : Effect.void, - ), Effect.catch((cause) => output.raw(`Failed to destroy shadow stack ${stack.id}: ${cause.message}.\n`, "stderr"), ), diff --git a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md index 581c6bfd6e..78d2a0a996 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md @@ -48,9 +48,9 @@ shutdown is refused. When no owner is running and the engine reports that its daemon cannot be reached, destruction removes nothing: the registration, port claims and host -data stay as they are, stderr warns that the stack's engine (Docker or Podman) resources were not removed, and -the command exits 0 reporting the stack as not destroyed. Run destroy again once -the engine is reachable. Any other engine failure, an owner starting during +data stay as they are, and the command exits 1 with one error whose suggestion +gives the exact `supabase stack destroy --stack-id --yes` command to run +once the engine is reachable. Any other engine failure, an owner starting during destruction, an instance whose data (including a native database's socket directory) cannot be removed, or a labelled container that survives removal, fails the command and keeps the stack registered; the error lists what @@ -69,8 +69,8 @@ routing/settings can read project configuration and profiles. Destruction releases this stack's rows in the per-user port registry at `/.supabase/ports.sqlite` (independent of `SUPABASE_HOME`, with no -override) only once cleanup is confirmed complete; a destroy that fails or skips -engine cleanup keeps the stack's reservations, and its ports remain unavailable +override) only once cleanup is confirmed complete; a destroy that fails +keeps the stack's reservations, and its ports remain unavailable to other stacks until a later destroy succeeds. A reservation whose owning stack's state was deleted without going through destroy (for example a removed or unmounted state root) is usually released by that stack's own still-running @@ -81,12 +81,15 @@ remains the backstop when no owner is left alive to do so. ## Output, exit codes and telemetry Text prints `Stack destroyed.` (`Removed the containers stack left behind.` -for a deleted stack's containers), or, when engine cleanup was skipped, -`Stack could not be fully destroyed because is unreachable; restore it and run "supabase stack destroy --stack-id " again.` -JSON and stream-json success data contain `destroyed` (`false` when engine -cleanup was skipped), `id`, and `runtime_cleanup` (`complete` or `skipped`); a -skipped cleanup also carries `engine`. Exit 0 on destruction or skipped engine cleanup, 1 on invalid flags, missing -selection, rejected/cancelled confirmation or cleanup failure, and 130 on -interruption. Standard command telemetry is unchanged, with no custom events. +for a deleted stack's containers). JSON and stream-json success data contain +`destroyed` (`true`) and `id`. Exit 0 on destruction, 1 on invalid flags, missing +selection, rejected/cancelled confirmation, an unreachable engine, or cleanup +failure, and 130 on interruption. A saved stack whose state cannot be decoded +(for example one saved by an older CLI with a service kind this CLI no longer +knows) fails with an error naming its directory under the state root; removing +that directory discards the stack. Its leftover containers are then removed +when a container stack next starts under the same state root or by +`destroy --stack-id`, and its port reservations are released when another stack +needs them; its data in the shared database volume is not removed. Standard command telemetry is unchanged, with no custom events. Telemetry flushes on success and failure to `/telemetry.json`. diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts index 537fa32228..030feafbe7 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts @@ -10,21 +10,26 @@ import { isStackId, StackApi, rejectStackOutput, - skippedRuntimeCleanupWarning, StackTargetResolver, mapTargetError, validateStackTarget, } from "../stack.shared.ts"; -import { containerEngineName } from "../../../../command-internal/stack-runtime.ts"; import type { StackDestroyFlags } from "./destroy.command.ts"; import { StackCommandDestroyError } from "./destroy.errors.ts"; -const destroyError = (cause: StackError) => - new StackCommandDestroyError({ - reason: "unknown", - message: cause.message, - cause, - }); +const destroyError = (id: string) => (cause: StackError) => + cause.reason === "runtime-unavailable" + ? new StackCommandDestroyError({ + reason: "runtime", + message: cause.message, + suggestion: `Start the container engine, then run "supabase stack destroy --stack-id ${id} --yes" again; nothing was removed.`, + cause, + }) + : new StackCommandDestroyError({ + reason: "unknown", + message: cause.message, + cause, + }); export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( flags: StackDestroyFlags, @@ -113,9 +118,9 @@ export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( } const stack = Option.isSome(deleted) ? deleted.value - : yield* api.open({ ...locations, id }).pipe(Effect.mapError(destroyError)); + : yield* api.open({ ...locations, id }).pipe(Effect.mapError(destroyError(id))); const destroying = yield* output.task(`Destroying stack ${id}...`); - const result = yield* stack.destroy.pipe( + yield* stack.destroy.pipe( Effect.onExit((exit) => Exit.isSuccess(exit) ? destroying.clear @@ -123,31 +128,16 @@ export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( ? destroying.cancel() : destroying.fail(Option.getOrUndefined(Exit.findErrorOption(exit))?.message), ), - Effect.mapError(destroyError), + Effect.mapError(destroyError(id)), ); - yield* Effect.annotateCurrentSpan({ - "stack.prompted": !yes, - "stack.runtime_cleanup": result.runtimeCleanup, - }); - if (result.runtimeCleanup === "skipped") - yield* output.warn(skippedRuntimeCleanupWarning(`stack ${id}`, id, result.engine)); - if (output.format !== "text") - yield* output.success("", { - destroyed: result.runtimeCleanup === "complete", - id, - runtime_cleanup: result.runtimeCleanup, - ...(result.runtimeCleanup === "skipped" ? { engine: result.engine } : {}), - }); - else if (result.runtimeCleanup === "complete") + yield* Effect.annotateCurrentSpan({ "stack.prompted": !yes }); + if (output.format !== "text") yield* output.success("", { destroyed: true, id }); + else yield* output.raw( target === undefined ? `Removed the containers stack ${id} left behind.\n` : `Stack ${id} destroyed.\n`, ); - else - yield* output.raw( - `Stack ${id} could not be fully destroyed because ${containerEngineName(result.engine)} is unreachable; restore it and run "supabase stack destroy --stack-id ${id}" again.\n`, - ); }); return yield* body.pipe(Effect.ensuring(telemetryState.flush)); }); diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts index e3465db0bb..b70ceab990 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts @@ -120,6 +120,25 @@ describe("stack destroy", () => { }).pipe(Effect.provide(live)), ); + it.live("refuses a stack with an unknown service kind and names its directory", () => + Effect.gen(function* () { + const f = yield* fixture(true); + const statePath = f.path.join(f.locations.stateRoot, f.stack.id, "state.json"); + const saved = yield* f.fs.readFileString(statePath); + yield* f.fs.writeFileString( + statePath, + saved.replace('"instances":[]', '"instances":[{"id":"logs","service":"vector"}]'), + ); + + const error = yield* stackDestroy(f.flags).pipe(Effect.provide(f.layer), Effect.flip); + + expect(error.message).toContain( + `Remove its directory ${f.path.join(f.locations.stateRoot, f.stack.id)}`, + ); + expect(yield* f.fs.exists(statePath)).toBe(true); + }).pipe(Effect.provide(live)), + ); + it.live("destroys only the stack addressed by the short ID that stack list shows", () => Effect.gen(function* () { const f = yield* fixture(true); diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts index 691e822f8a..10dc8d656c 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts @@ -1,6 +1,7 @@ import { BunServices } from "@effect/platform-bun"; import { expect, it } from "@effect/vitest"; import { Effect, FileSystem, Layer, Option, Path } from "effect"; +import { StackError } from "@supabase/stack/effect"; import { CliArgs } from "../../../../shared/cli/cli-args.service.ts"; import { YesFlag } from "../../../../command-internal/global-flags.ts"; import { @@ -10,32 +11,37 @@ import { import { mockOutput, mockStdin, mockTty } from "../../../../../tests/helpers/mocks.ts"; import { StackApi, stackApiLayer, stackTargetResolverLayer } from "../stack.shared.ts"; import { stackDestroy } from "./destroy.handler.ts"; +import { StackCommandDestroyError } from "./destroy.errors.ts"; const live = Layer.provideMerge(stackApiLayer, BunServices.layer); /** - * A registration real enough for target resolution, whose `destroy` reports the skipped-engine - * outcome directly: the namespace reconcile this models (claim stays, a later acquisition - * finishes it) is covered by packages/stack's own suite; this fixture is only the CLI's - * formatting of that result. + * A real registration whose `destroy` fails the way the stack owner reports an unreachable engine: + * the namespace behaviour behind it (registration and claims untouched) is covered by + * packages/stack's own suite; this fixture is only the CLI's mapping of that failure. */ const fixture = Effect.fn("StackDestroyRuntimeUnavailableTest.fixture")(function* ( format: "text" | "json", - engine: "docker" | "podman" = "docker", ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-runtime-" }); const realApi = yield* StackApi; const locations = { stateRoot: path.join(root, "stacks"), cacheRoot: path.join(root, "cache") }; - const stack = yield* realApi.create({ ...locations, projectRoot: root, runtime: engine }); + const stack = yield* realApi.create({ ...locations, projectRoot: root, runtime: "docker" }); const api = StackApi.of({ ...realApi, open: (options) => realApi.open(options).pipe( Effect.map((opened) => ({ ...opened, - destroy: Effect.succeed({ runtimeCleanup: "skipped", engine } as const), + destroy: Effect.fail( + new StackError({ + operation: "destroy", + message: "Docker CLI or daemon isn't reachable", + reason: "runtime-unavailable", + }), + ), })), ), }); @@ -65,52 +71,22 @@ const fixture = Effect.fn("StackDestroyRuntimeUnavailableTest.fixture")(function }; }); -it.live( - "keeps a stack registered and reports pending engine cleanup when its engine is unreachable", - () => - Effect.gen(function* () { - const f = yield* fixture("text"); - yield* stackDestroy(f.flags).pipe(Effect.provide(f.layer)); - expect(f.output.stdoutText).toBe( - `Stack ${f.stack.id} could not be fully destroyed because Docker is unreachable; restore it and run "supabase stack destroy --stack-id ${f.stack.id}" again.\n`, - ); - expect(f.output.messages).toContainEqual({ - type: "warn", - message: `Docker was unavailable, so Docker resources for stack ${f.stack.id} were not removed. Restore Docker and run "supabase stack destroy --stack-id ${f.stack.id}" again to finish removing it.`, - }); - expect(yield* f.api.discover(f.locations)).toHaveLength(1); - }).pipe(Effect.scoped, Effect.provide(live)), -); +for (const format of ["text", "json"] as const) + it.live( + `fails with the exact --yes retry command and keeps the stack registered when its engine is unreachable (${format})`, + () => + Effect.gen(function* () { + const f = yield* fixture(format); -it.live("reports skipped engine cleanup in the JSON result", () => - Effect.gen(function* () { - const f = yield* fixture("json"); - yield* stackDestroy(f.flags).pipe(Effect.provide(f.layer)); - expect(f.output.messages).toContainEqual( - expect.objectContaining({ - data: { - destroyed: false, - id: f.stack.id, - runtime_cleanup: "skipped", - engine: "docker", - }, - }), - ); - expect(yield* f.api.discover(f.locations)).toHaveLength(1); - }).pipe(Effect.scoped, Effect.provide(live)), -); + const failure = yield* stackDestroy(f.flags).pipe(Effect.provide(f.layer), Effect.flip); -it.live("names Podman when a Podman stack's engine is unreachable", () => - Effect.gen(function* () { - const f = yield* fixture("text", "podman"); - yield* stackDestroy(f.flags).pipe(Effect.provide(f.layer)); - expect(f.output.stdoutText).toBe( - `Stack ${f.stack.id} could not be fully destroyed because Podman is unreachable; restore it and run "supabase stack destroy --stack-id ${f.stack.id}" again.\n`, - ); - expect(f.output.messages).toContainEqual({ - type: "warn", - message: `Podman was unavailable, so Podman resources for stack ${f.stack.id} were not removed. Restore Podman and run "supabase stack destroy --stack-id ${f.stack.id}" again to finish removing it.`, - }); - expect(yield* f.api.discover(f.locations)).toHaveLength(1); - }).pipe(Effect.scoped, Effect.provide(live)), -); + expect(failure).toBeInstanceOf(StackCommandDestroyError); + expect(failure.reason).toBe("runtime"); + expect(failure.suggestion).toContain( + `supabase stack destroy --stack-id ${f.stack.id} --yes`, + ); + expect(f.output.stdoutText).not.toContain("destroyed"); + expect(f.output.messages.filter(({ type }) => type === "success")).toEqual([]); + expect(yield* f.api.discover(f.locations)).toHaveLength(1); + }).pipe(Effect.scoped, Effect.provide(live)), + ); diff --git a/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts index ffa34116e0..b307ec32c9 100644 --- a/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts @@ -233,6 +233,38 @@ describe("stack target resolver", () => { }).pipe(Effect.provide(BunServices.layer)), ); + it.live( + "tells the user to remove the directory of a saved stack with an unknown service kind", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { root, home, register, resolve } = yield* workspace; + const id = yield* register({ projectRoot: root }); + const statePath = path.join(home, "stacks", id, "state.json"); + const saved = yield* fs.readFileString(statePath); + yield* fs.writeFileString( + statePath, + saved.replace('"instances":[]', '"instances":[{"id":"logs","service":"vector"}]'), + ); + + const byProject = yield* resolve({ projectRoot: root, runtime: "auto" }).pipe(Effect.flip); + const byPrefix = yield* resolve({ + projectRoot: root, + id: id.slice(0, 8), + runtime: "auto", + }).pipe(Effect.flip); + + for (const failure of [byProject, byPrefix]) { + expect(failure.reason).toBe("invalid-config"); + expect(failure.message).toContain(`Stack ${id} could not be read`); + expect(failure.message).toContain( + `Remove its directory ${path.join(home, "stacks", id)}`, + ); + } + }).pipe(Effect.provide(BunServices.layer)), + ); + it.live("surfaces an unreadable saved stack that an id prefix selects", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/cli/src/commands/experimental/stack/stack.shared.ts b/apps/cli/src/commands/experimental/stack/stack.shared.ts index 7afd6dab95..d924692707 100644 --- a/apps/cli/src/commands/experimental/stack/stack.shared.ts +++ b/apps/cli/src/commands/experimental/stack/stack.shared.ts @@ -11,17 +11,13 @@ import { type CliErrorActionabilityDeclaration, ErrorActionabilityId, } from "../../../shared/telemetry/error-actionability.ts"; -import { - skippedRuntimeCleanupWarning, - StackApi, - stackApiLayer, -} from "../../../command-internal/stack-api.ts"; +import { StackApi, stackApiLayer } from "../../../command-internal/stack-api.ts"; import { recordStackRuntimeTelemetry, type StackRuntime, } from "../../../command-internal/stack-runtime.ts"; -export { skippedRuntimeCleanupWarning, StackApi, stackApiLayer }; +export { StackApi, stackApiLayer }; /** The target selected by the CLI adapter for one stack command. */ export interface StackTarget { @@ -162,13 +158,7 @@ export const stackTargetResolverLayer = Layer.effect( }); const [invalid] = unreadable; if (invalid !== undefined) - return yield* new StackTargetError({ - message: `Stack ${invalid.id} could not be read: ${invalid.error.message}`, - reason: "invalid-config", - suggestion: - "Inspect the stack registry under $SUPABASE_HOME/stacks or ~/.supabase/stacks.", - cause: invalid.error, - }); + yield* stackApi.find({ stateRoot, id: invalid.id }).pipe(Effect.mapError(stateError)); return matches[0]; }); const resolve = Effect.fn("StackTargetResolver.resolve")(function* (input: { diff --git a/packages/stack/src/destroy.runtime-unavailable.integration.test.ts b/packages/stack/src/destroy.runtime-unavailable.integration.test.ts index 1611c4f512..faeaa69e60 100644 --- a/packages/stack/src/destroy.runtime-unavailable.integration.test.ts +++ b/packages/stack/src/destroy.runtime-unavailable.integration.test.ts @@ -43,7 +43,7 @@ const shimDocker = Effect.fn("shimDocker")(function* (root: string, script: stri }); it.live( - "keeps a stack registered when destroy finds no owner and its engine is unreachable, then finishes the cleanup once a later owner can start", + "fails destroy with runtime-unavailable and keeps the stack registered when its engine is unreachable, then finishes the cleanup once a later owner can start", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -61,8 +61,8 @@ it.live( } satisfies Parameters[0]; const stack = yield* create(options); - const result = yield* stack.destroy; - expect(result).toEqual({ runtimeCleanup: "skipped", engine: "docker" }); + const failure = yield* Effect.flip(stack.destroy); + expect(failure.reason).toBe("runtime-unavailable"); expect(yield* discover({ stateRoot: options.stateRoot })).toHaveLength(1); // The engine is back: destroy retried finishes the cleanup automatically. @@ -93,9 +93,9 @@ it.live("keeps a stack registered when Windows reports its Docker daemon pipe is runtime: "docker", }); - const result = yield* stack.destroy; + const failure = yield* Effect.flip(stack.destroy); - expect(result.runtimeCleanup).toBe("skipped"); + expect(failure.reason).toBe("runtime-unavailable"); expect(yield* discover({ stateRoot })).toHaveLength(1); }).pipe(Effect.scoped, Effect.provide(layer)), ); @@ -123,9 +123,9 @@ it.live("keeps a stack registered when its engine CLI is not installed", () => }), ); - const result = yield* stack.destroy; + const failure = yield* Effect.flip(stack.destroy); - expect(result.runtimeCleanup).toBe("skipped"); + expect(failure.reason).toBe("runtime-unavailable"); expect(yield* discover({ stateRoot })).toHaveLength(1); }).pipe(Effect.scoped, Effect.provide(layer)), ); @@ -146,9 +146,9 @@ it.live("keeps a stack registered when Docker reports its API socket is missing" runtime: "docker", }); - const result = yield* stack.destroy; + const failure = yield* Effect.flip(stack.destroy); - expect(result.runtimeCleanup).toBe("skipped"); + expect(failure.reason).toBe("runtime-unavailable"); expect(yield* discover({ stateRoot })).toHaveLength(1); }).pipe(Effect.scoped, Effect.provide(layer)), ); diff --git a/packages/stack/src/effect.integration.test.ts b/packages/stack/src/effect.integration.test.ts index 6243baf069..05f447cee8 100644 --- a/packages/stack/src/effect.integration.test.ts +++ b/packages/stack/src/effect.integration.test.ts @@ -670,6 +670,35 @@ it.live("finds a deleted stack only by the containers left in its own state root }).pipe(Effect.scoped, Effect.provide(layer)), ); +it.live("names the directory of a saved stack whose state cannot be decoded", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-unreadable-" }); + const options = { + projectRoot: root, + stateRoot: `${root}/state`, + cacheRoot: `${root}/cache`, + runtime: "native", + } satisfies Parameters[0]; + const stack = yield* create(options); + const statePath = `${options.stateRoot}/${stack.id}/state.json`; + const saved = yield* fs.readFileString(statePath); + yield* fs.writeFileString( + statePath, + saved.replace('"instances":[]', '"instances":[{"id":"logs","service":"vector"}]'), + ); + + const failure = yield* find({ stateRoot: options.stateRoot, projectRoot: root }).pipe( + Effect.flip, + ); + + expect(failure.operation).toBe("find"); + expect(failure.message).toContain(`Stack ${stack.id} could not be read`); + expect(failure.message).toContain(`Remove its directory ${options.stateRoot}/${stack.id}`); + expect(yield* fs.exists(statePath)).toBe(true); + }).pipe(Effect.scoped, Effect.provide(layer)), +); + it.live("removes nothing for a stack whose registration is unreadable", () => Effect.gen(function* () { const { fs, locations, containers } = yield* deletedStackRoot(); diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index e08cebedbc..fff59827dc 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -200,14 +200,11 @@ export type ServiceInstances = { }; type AnyInstance = ServiceInstances[Kind]; /** - * The outcome of {@link Stack.destroy}. `skipped` means the stack's container engine was - * unreachable, so nothing was confirmed removed: its registration, data and claimed resources all - * stay in place. Recovery means calling destroy again once the engine is reachable; reopening the - * stack does not clean this up on its own. + * The outcome of {@link Stack.destroy}. When the stack's container engine is unreachable, destroy + * fails with reason `runtime-unavailable` and leaves the registration, data and claimed resources + * in place; calling it again once the engine is reachable finishes the cleanup. */ -export type DestroyResult = - | { readonly runtimeCleanup: "complete" } - | { readonly runtimeCleanup: "skipped"; readonly engine: "docker" | "podman" }; +export type DestroyResult = { readonly runtimeCleanup: "complete" }; /** Options for streaming PostgreSQL command input and output. */ export interface PostgresCommandOptions { readonly args?: ReadonlyArray; @@ -310,27 +307,6 @@ interface Connection { /** Replaced by a newer connection; closes once its last user finishes. */ retired: boolean; } -/** - * Destroys a stack whose owner cannot start because its container engine is unreachable: since - * nothing can be confirmed removed, registration, data and claims all stay exactly as they are. - * The next lease acquisition's reconcile loop (and, once an owner can start, the stack's normal - * destroy path) finishes the cleanup once the engine is reachable again. - */ -const destroyWithoutEngine = Effect.fn("Stack.destroyWithoutEngine")( - function* (state: StackNamespace.Interface, saved: SavedStack, engine: "docker" | "podman") { - const id = saved.id; - if (yield* state.leased(id)) - return yield* failure( - "destroy", - "An owner for this stack started during destroy; run destroy again", - ); - const current = yield* state.read(id); - if (current === undefined) return { runtimeCleanup: "complete" } as const; - return { runtimeCleanup: "skipped", engine } as const; - }, - Effect.mapError((cause) => failure("destroy", cause)), -); - /** `launch` starts an owner when none is live; `attach` requires a live one. */ type Reach = "launch" | "attach"; @@ -516,11 +492,10 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( const shutdown = Effect.fn("Stack.shutdown")(function* (destroy: boolean) { const operation = destroy ? "destroy" : "shutdown"; yield* invalidate(); - const engine = saved.runtime === "native" ? undefined : saved.runtime; // Shutdown uses the release-stable endpoint, so it reaches owners of any release. - const { endpoint, refusal, engineUnavailable } = yield* Effect.scoped( + const { endpoint, refusal } = yield* Effect.scoped( Effect.gen(function* () { - const idle = { endpoint: undefined, refusal: Exit.void, engineUnavailable: false }; + const idle = { endpoint: undefined, refusal: Exit.void }; const live = yield* connectHost(state, saved.id, { anyRelease: true }).pipe( Effect.map(Option.some), Effect.catchIf(ownerAbsent, () => @@ -531,13 +506,7 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( }), ), Effect.catchIf(stackGone, () => Effect.succeed(Option.none())), - // The owner's startup sweep reports an unreachable engine before any owner serves. - Effect.catchIf( - (cause) => engine !== undefined && hasReason("runtime-unavailable")(cause), - () => Effect.succeed("engine-unavailable" as const), - ), ); - if (live === "engine-unavailable") return { ...idle, engineUnavailable: true }; if (Option.isNone(live)) return idle; const access = live.value; const endpoint = access.endpoint; @@ -557,16 +526,12 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( ), Effect.exit, ); - return { endpoint, refusal, engineUnavailable: false }; + return { endpoint, refusal }; }), ).pipe( Effect.provideContext(services), Effect.mapError((cause) => failure(operation, cause)), ); - if (engineUnavailable && engine !== undefined) - return yield* destroyWithoutEngine(state, saved, engine).pipe( - Effect.provideContext(services), - ); if (endpoint === undefined) return { runtimeCleanup: "complete" } as const; if (Exit.isFailure(refusal)) return yield* Option.match(Cause.findErrorOption(refusal.cause), { @@ -987,7 +952,17 @@ export const find = Effect.fn("Stack.find")( const state = yield* stateFor(options.stateRoot); const id = "id" in options ? options.id : yield* deriveStackId(yield* resolveStackIdentity(options)); - const definition = yield* state.read(id); + const path = yield* Path.Path; + const definition = yield* state.read(id).pipe( + Effect.catchIf( + (error) => error.operation === "decode", + (error) => + failure( + "find", + `Stack ${id} could not be read: ${error.message}. Remove its directory ${path.join(options.stateRoot, id)} to discard it. Containers it left behind are removed when a container stack next starts under this state root or when the stack is destroyed by id, and its port reservations are released when another stack needs them; its data in the shared database volume is not removed.`, + ), + ), + ); if (definition === undefined) return Option.none(); return Option.some({ definition, host: yield* observeHost(state, definition) }); }, From 0e65ab31bc17f64edb069d1313c364d44a146d69 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 14:32:45 +0200 Subject: [PATCH 128/142] refactor(stack): make stack destroy succeed with no result value --- .../commands/db/dump/dump.integration.test.ts | 2 +- .../commands/db/start/start.integration.test.ts | 2 +- .../stack/destroy/destroy.integration.test.ts | 1 - .../stack/prepare/prepare.integration.test.ts | 2 +- .../stack/start/start.integration.test.ts | 1 - ...troy.runtime-unavailable.integration.test.ts | 3 +-- packages/stack/src/effect.integration.test.ts | 2 +- packages/stack/src/effect.ts | 17 +++++++---------- packages/stack/src/index.ts | 1 - 9 files changed, 12 insertions(+), 19 deletions(-) diff --git a/apps/cli/src/commands/db/dump/dump.integration.test.ts b/apps/cli/src/commands/db/dump/dump.integration.test.ts index 7e482fc430..d9ed85edff 100644 --- a/apps/cli/src/commands/db/dump/dump.integration.test.ts +++ b/apps/cli/src/commands/db/dump/dump.integration.test.ts @@ -148,7 +148,7 @@ const managedDumpStackApi = (runtime: "native" | "docker") => { restart: Effect.succeed([]), }, stop: Effect.void, - destroy: Effect.succeed({ runtimeCleanup: "complete" as const }), + destroy: Effect.void, commands: { run: runCommand }, } satisfies Stack; return Layer.succeed(StackApi, { diff --git a/apps/cli/src/commands/db/start/start.integration.test.ts b/apps/cli/src/commands/db/start/start.integration.test.ts index 811178125c..58d1775b72 100644 --- a/apps/cli/src/commands/db/start/start.integration.test.ts +++ b/apps/cli/src/commands/db/start/start.integration.test.ts @@ -1729,7 +1729,7 @@ describe("db start stack backend", () => { restart: Effect.succeed([]), }, stop: Effect.void, - destroy: Effect.succeed({ runtimeCleanup: "complete" as const }), + destroy: Effect.void, commands: { run: () => Effect.die("unused") }, }; return { stack, state }; diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts index b70ceab990..5613044551 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts @@ -174,7 +174,6 @@ describe("stack destroy", () => { id, destroy: Effect.sync(() => { destroyed += 1; - return { runtimeCleanup: "complete" } as const; }), }) : Option.none(), diff --git a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts index fc38702510..109ebc6ae2 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts @@ -154,7 +154,7 @@ const makeFixture = (root: string, options: FixtureOptions = {}) => { restart: Effect.succeed([]), }, stop: Effect.void, - destroy: Effect.succeed({ runtimeCleanup: "complete" as const }), + destroy: Effect.void, commands: { run: () => Effect.die("unused") }, }; const output = mockOutput(); diff --git a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts index 0e622fead3..8a7ba87809 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts @@ -402,7 +402,6 @@ const fakeStack = (compositionStart?: Stack["composition"]["start"]) => { }), destroy: Effect.sync(() => { hostDestroyed += 1; - return { runtimeCleanup: "complete" as const }; }), commands: { run: () => Effect.die("command not used") }, }; diff --git a/packages/stack/src/destroy.runtime-unavailable.integration.test.ts b/packages/stack/src/destroy.runtime-unavailable.integration.test.ts index faeaa69e60..8db341a4e5 100644 --- a/packages/stack/src/destroy.runtime-unavailable.integration.test.ts +++ b/packages/stack/src/destroy.runtime-unavailable.integration.test.ts @@ -70,8 +70,7 @@ it.live( root, '#!/bin/sh\nif [ "$1" = "info" ]; then\n echo \'fake-daemon-id\'\nfi\nexit 0\n', ); - const second = yield* stack.destroy; - expect(second).toEqual({ runtimeCleanup: "complete" }); + yield* stack.destroy; expect(yield* discover({ stateRoot: options.stateRoot })).toEqual([]); }).pipe(Effect.scoped, Effect.provide(layer)), 30_000, diff --git a/packages/stack/src/effect.integration.test.ts b/packages/stack/src/effect.integration.test.ts index 05f447cee8..2a2011490d 100644 --- a/packages/stack/src/effect.integration.test.ts +++ b/packages/stack/src/effect.integration.test.ts @@ -757,7 +757,7 @@ it.live("removes a deleted stack's containers from both engines, not another roo yield* findDeleted({ ...locations, id: deletedId }).pipe(Effect.provide(engine.layer)), ); - expect(yield* deleted.destroy).toEqual({ runtimeCleanup: "complete" }); + yield* deleted.destroy; const kept = containers.filter(({ id }) => id !== "deleted"); expect(engine.remaining("docker")).toEqual(kept); expect(engine.remaining("podman")).toEqual(kept); diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index fff59827dc..f86d39bd63 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -199,12 +199,6 @@ export type ServiceInstances = { [K in Kind]: K extends "database" ? DatabaseInstance : ServiceInstance; }; type AnyInstance = ServiceInstances[Kind]; -/** - * The outcome of {@link Stack.destroy}. When the stack's container engine is unreachable, destroy - * fails with reason `runtime-unavailable` and leaves the registration, data and claimed resources - * in place; calling it again once the engine is reachable finishes the cleanup. - */ -export type DestroyResult = { readonly runtimeCleanup: "complete" }; /** Options for streaming PostgreSQL command input and output. */ export interface PostgresCommandOptions { readonly args?: ReadonlyArray; @@ -270,7 +264,12 @@ export interface Stack { readonly restart: Effect.Effect, StackError>; }; readonly stop: Effect.Effect; - readonly destroy: Effect.Effect; + /** + * Removes the stack. When its container engine is unreachable, fails with reason + * `runtime-unavailable` and leaves the registration, data and claimed resources in place; + * calling it again once the engine is reachable finishes the cleanup. + */ + readonly destroy: Effect.Effect; readonly commands: { readonly run: CommandRunner; }; @@ -532,7 +531,7 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( Effect.provideContext(services), Effect.mapError((cause) => failure(operation, cause)), ); - if (endpoint === undefined) return { runtimeCleanup: "complete" } as const; + if (endpoint === undefined) return; if (Exit.isFailure(refusal)) return yield* Option.match(Cause.findErrorOption(refusal.cause), { onNone: () => failure(operation, Cause.pretty(refusal.cause)), @@ -541,7 +540,6 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( yield* waitForOwnerExit(endpoint.pid, ownerExitProbe(fs)).pipe( Effect.mapError((cause) => failure("shutdown-exit", cause)), ); - return { runtimeCleanup: "complete" } as const; }); const snapshotScope = (options: DatabaseSnapshotOptions | undefined) => @@ -1040,7 +1038,6 @@ export const findDeleted = Effect.fn("Stack.findDeleted")( "destroy", `Removed the ${matched.map(({ target }) => engineLabel(target.engine)).join(" and ")} containers stack ${options.id} left behind. ${unlisted}`, ); - return { runtimeCleanup: "complete" } as const; }).pipe( Effect.mapError((cause) => failure("destroy", cause)), Effect.provideContext(services), diff --git a/packages/stack/src/index.ts b/packages/stack/src/index.ts index db15199cb1..5b7774723e 100644 --- a/packages/stack/src/index.ts +++ b/packages/stack/src/index.ts @@ -21,7 +21,6 @@ export type { CreateOptions, CreationChange, DatabaseSnapshotOptions, - DestroyResult, FindOptions, FoundStack, LogPosition, From ae3854a153cf32276f2cde0770adbe212b57f56a Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 14:37:09 +0200 Subject: [PATCH 129/142] fix(stack): hold pinned port reservations only while their listener is open --- .../start/start.native.integration.test.ts | 2 +- .../stack/src/Network.integration.test.ts | 105 ++++++++++++- packages/stack/src/Network.ts | 106 +++++++------ packages/stack/src/Ports.integration.test.ts | 146 ++++++++++++++++++ packages/stack/src/Ports.ts | 107 +++++++++---- 5 files changed, 389 insertions(+), 77 deletions(-) diff --git a/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts index 85363964ec..b7ffa80e82 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts @@ -381,7 +381,7 @@ describe("experimental stack start native lifecycle", () => { // Linux rejects the bind; platforms that allow overlapping binds reject the probe. expect(Cause.pretty(failedStart.cause)).toMatch( new RegExp( - `127\\.0\\.0\\.1:${occupiedPort}(: Cannot bind TCP listener| is already in use)`, + `127\\.0\\.0\\.1:${occupiedPort}(: Cannot bind TCP listener| is in use by another process)`, ), ); const ownerPid = ownerPids[attempt]; diff --git a/packages/stack/src/Network.integration.test.ts b/packages/stack/src/Network.integration.test.ts index 8e0719b623..188c166e3a 100644 --- a/packages/stack/src/Network.integration.test.ts +++ b/packages/stack/src/Network.integration.test.ts @@ -1,6 +1,7 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Context, Data, Deferred, Effect, Fiber, FileSystem, Layer, Path, Ref } from "effect"; +import { Context, Data, Deferred, Effect, Fiber, FileSystem, Hash, Layer, Path, Ref } from "effect"; +import { randomUUID } from "node:crypto"; import * as Net from "node:net"; // oxlint-disable-line effecttsgo/node-builtin-import -- fixture retains an idle HTTP connection. import { createServer } from "node:http"; // oxlint-disable-line effecttsgo/node-builtin-import -- real socket fixture. import { HttpClient } from "effect/unstable/http"; @@ -218,6 +219,7 @@ it.live("reports a dedicated port conflict without taking over the holder's rese const state = yield* makeTestState(root); yield* state.save(stack("first")); yield* state.save(stack("second")); + yield* state.acquireLease("first"); const target = yield* backend; const firstEnabled = yield* Ref.make(true); const firstNetwork = yield* makeTestNetwork({ stackId: "first", runtime: "native", state }); @@ -594,3 +596,104 @@ it.live("addresses docker runtime endpoints through the stack host alias", () => }), ).pipe(Effect.provide(NodeServices.layer)), ); + +/** A pinned port above the auto range, so it never meets a registry row from an auto allocation. */ +const pinnedPort = () => 33000 + (Math.abs(Hash.string(randomUUID())) % 16000); + +it.live("addresses a pinned endpoint from its configured port without a registry row", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-pinned-address-" }); + const state = yield* makeTestState(root); + yield* state.save(stack("stack")); + const target = yield* backend; + const port = pinnedPort(); + const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); + const namespace = yield* network.register({ + id: "db", + endpoints: { sql: { ...endpoint(target, Effect.succeed(false)), port } }, + }); + + expect((yield* namespace.address("sql", "host")).port).toBe(port); + expect((yield* namespace.address("sql", "runtime")).port).toBe(port); + expect(yield* reservedPort(root, "stack", "db:sql")).toBeUndefined(); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("a pinned port is reserved only while its listener is open", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-pinned-stop-" }); + const state = yield* makeTestState(root); + yield* state.save(stack("first")); + yield* state.save(stack("second")); + const target = yield* backend; + const port = pinnedPort(); + const firstEnabled = yield* Ref.make(true); + const firstNetwork = yield* makeTestNetwork({ stackId: "first", runtime: "native", state }); + const first = yield* firstNetwork.register({ + id: "db", + endpoints: { sql: { ...endpoint(target, Ref.get(firstEnabled)), port } }, + }); + yield* first.bind; + expect(yield* reservedPort(root, "first", "db:sql")).toBe(port); + + yield* Ref.set(firstEnabled, false); + yield* first.close; + expect(yield* reservedPort(root, "first", "db:sql")).toBeUndefined(); + + const secondNetwork = yield* makeTestNetwork({ stackId: "second", runtime: "native", state }); + const second = yield* secondNetwork.register({ + id: "db", + endpoints: { sql: { ...endpoint(target, Effect.succeed(true)), port } }, + }); + yield* second.bind; + expect((yield* second.address("sql", "host")).port).toBe(port); + expect(yield* request("127.0.0.1", port, "/started")).toBe("backend:/started"); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("a pinned shared listener keeps its reservation until its last route closes", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-pinned-shared-" }); + const state = yield* makeTestState(root); + yield* state.save(stack("stack")); + const target = yield* backend; + const port = pinnedPort(); + const firstEnabled = yield* Ref.make(true); + const secondEnabled = yield* Ref.make(true); + const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); + const first = yield* network.register({ + id: "rest", + endpoints: { + api: { ...endpoint(target, Ref.get(firstEnabled)), port, shared: [{ prefix: "/rest" }] }, + }, + }); + const second = yield* network.register({ + id: "auth", + endpoints: { + api: { ...endpoint(target, Ref.get(secondEnabled)), port, shared: [{ prefix: "/auth" }] }, + }, + }); + yield* first.bind; + yield* second.bind; + expect(yield* reservedPort(root, "stack", "api")).toBe(port); + expect(yield* request("127.0.0.1", port, "/auth/v1")).toBe("backend:/auth/v1"); + + yield* Ref.set(firstEnabled, false); + yield* first.close; + expect(yield* reservedPort(root, "stack", "api")).toBe(port); + expect(yield* request("127.0.0.1", port, "/auth/v1")).toBe("backend:/auth/v1"); + + yield* Ref.set(secondEnabled, false); + yield* second.close; + expect(yield* reservedPort(root, "stack", "api")).toBeUndefined(); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index 7318b244b2..25b9d71abf 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -2,7 +2,7 @@ import { Context, Data, Effect, Exit, Layer, Ref, Scope, Semaphore } from "effec import { DOCKER_HOST_ALIAS } from "./runtime/Container.ts"; import * as PortReservations from "./namespace/PortReservations.ts"; import * as StackNamespace from "./StackNamespace.ts"; -import { makePorts, PortError, probeVacant } from "./Ports.ts"; +import { makePorts, probeVacant } from "./Ports.ts"; import { bindTcp, serveTcp, type BackendAddress, type ProxyError } from "./Proxy.ts"; import { makeHttpProxy, type HttpProxy, type HttpRoute } from "./HttpProxy.ts"; @@ -71,6 +71,15 @@ const errorFor = (operation: string, cause: unknown) => cause, }); +/** A later claimant reuses the open shared listener, so it never touches the port registry. */ +const joinClaim = ( + existing: { readonly claim: number; readonly proxy: HttpProxy }, + port: number | "auto", +) => + port !== "auto" && port !== existing.claim + ? Effect.fail(errorFor("bind", "Shared API port differs from existing claim")) + : Effect.succeed({ port: existing.claim, listener: { proxy: existing.proxy } }); + const makeNetwork = (options: { readonly stackId: string; readonly runtime: NetworkRuntime; @@ -160,50 +169,50 @@ const makeNetwork = (options: { Effect.gen(function* () { const endpointScope = yield* Scope.fork(owner, "sequential"); const key = endpoint.shared === undefined ? `${id}:${name}` : "api"; + const claimed = + endpoint.shared === undefined ? undefined : yield* Ref.get(shared); const result = yield* restore( - ports - .acquire<{ readonly proxy?: HttpProxy }, Scope.Scope>( - { stackId: options.stackId, key, host: listenHost, port: endpoint.port }, - (host, port) => - Effect.gen(function* () { - if (endpoint.shared !== undefined) { - const current = yield* Ref.get(shared); - if (current !== undefined) { - if (current.claim !== port) - return yield* new PortError({ - key: "api", - message: "Shared API port differs from existing claim", - }); - return { proxy: current.proxy }; - } - yield* probe(key, host, port); - const proxy = yield* makeHttpProxy({ host, port }); - return { proxy }; - } - if (endpoint.protocol === "http") { - yield* probe(key, host, port); - const proxy = yield* makeHttpProxy({ host, port }); - yield* proxy.setRoutes([ - { id, prefix: "/", target: endpoint.backend }, - ]); - return { proxy }; - } - yield* probe(key, host, port); - const listener = yield* bindTcp(host, port); - yield* Effect.forkIn( - serveTcp(listener, endpoint.backend, `${id}:${name}`).pipe( - Effect.provideService(Scope.Scope, endpointScope), - ), - endpointScope, - { startImmediately: true }, - ); - return {}; - }), - ) - .pipe( - Effect.provideService(Scope.Scope, endpointScope), - Effect.mapError((cause) => errorFor("bind", cause)), - ), + claimed !== undefined + ? joinClaim(claimed, endpoint.port) + : ports + .acquire<{ readonly proxy?: HttpProxy }, Scope.Scope>( + { + stackId: options.stackId, + key, + host: listenHost, + port: endpoint.port, + }, + (host, port) => + Effect.gen(function* () { + if (endpoint.shared !== undefined) { + yield* probe(key, host, port); + const proxy = yield* makeHttpProxy({ host, port }); + return { proxy }; + } + if (endpoint.protocol === "http") { + yield* probe(key, host, port); + const proxy = yield* makeHttpProxy({ host, port }); + yield* proxy.setRoutes([ + { id, prefix: "/", target: endpoint.backend }, + ]); + return { proxy }; + } + yield* probe(key, host, port); + const listener = yield* bindTcp(host, port); + yield* Effect.forkIn( + serveTcp(listener, endpoint.backend, `${id}:${name}`).pipe( + Effect.provideService(Scope.Scope, endpointScope), + ), + endpointScope, + { startImmediately: true }, + ); + return {}; + }), + ) + .pipe( + Effect.provideService(Scope.Scope, endpointScope), + Effect.mapError((cause) => errorFor("bind", cause)), + ), ).pipe( Effect.onExit((exit) => Exit.isFailure(exit) ? Scope.close(endpointScope, exit) : Effect.void, @@ -314,9 +323,12 @@ const makeNetwork = (options: { const endpoint = endpoints[name]; if (endpoint === undefined) return yield* errorFor("address", `Unknown endpoint ${name}`); const key = endpoint.shared === undefined ? `${id}:${name}` : "api"; - const port = yield* ports - .assigned(options.stackId, key) - .pipe(Effect.mapError((cause) => errorFor("address", cause))); + const port = + endpoint.port === "auto" + ? yield* ports + .assigned(options.stackId, key) + .pipe(Effect.mapError((cause) => errorFor("address", cause))) + : endpoint.port; if (port === undefined) return yield* errorFor("address", `Endpoint ${name} is not assigned`); return { diff --git a/packages/stack/src/Ports.integration.test.ts b/packages/stack/src/Ports.integration.test.ts index 09affd1598..411ba9f721 100644 --- a/packages/stack/src/Ports.integration.test.ts +++ b/packages/stack/src/Ports.integration.test.ts @@ -486,3 +486,149 @@ it.live.skipIf(process.platform === "win32")( }), ).pipe(Effect.provide(NodeServices.layer)), ); + +/** A pinned port above the auto range, so it never meets a registry row from an auto allocation. */ +const pinnedPort = () => 33000 + (Math.abs(Hash.string(randomUUID())) % 16000); + +const withRegistry = Effect.provide( + Layer.merge(NodeServices.layer, PortReservations.layer.pipe(Layer.provide(NodeServices.layer))), +); + +/** Two stacks of separate projects registered in one state root. */ +const twoProjects = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const state = yield* makeTestState(root); + const projectA = `${root}/project-a`; + const projectB = `${root}/project-b`; + const a = `pinned-a-${randomUUID()}`; + const b = `pinned-b-${randomUUID()}`; + yield* saveStack(state, projectA, a); + yield* saveStack(state, projectB, b); + const ports = yield* makePorts(state); + const registry = Context.get( + yield* Layer.build(PortReservations.layer), + PortReservations.Service, + ); + return { state, ports, registry, stateRoot: yield* fs.realPath(root), a, b, projectA }; +}); + +it.live("a stopped stack's pinned port is free for another project's stack", () => + Effect.scoped( + Effect.gen(function* () { + const { ports, registry, stateRoot, a, b } = yield* twoProjects; + const port = pinnedPort(); + const bind = (_host: string, bound: number) => Effect.succeed(bound); + + yield* Effect.scoped( + ports.acquire({ stackId: a, key: "db:sql", host: "127.0.0.1", port }, bind), + ); + + expect(yield* registry.find(stateRoot, a, "db:sql")).toBeUndefined(); + const started = yield* ports.acquire( + { stackId: b, key: "db:sql", host: "127.0.0.1", port }, + bind, + ); + expect(started.port).toBe(port); + }), + ).pipe(withRegistry), +); + +it.live("a running stack's pinned port conflict names the port, its project, and the fix", () => + Effect.scoped( + Effect.gen(function* () { + const { state, ports, a, b, projectA } = yield* twoProjects; + const port = pinnedPort(); + const bind = (_host: string, bound: number) => Effect.succeed(bound); + yield* state.acquireLease(a); + yield* ports.acquire({ stackId: a, key: "db:sql", host: "127.0.0.1", port }, bind); + + const failure = yield* ports + .acquire({ stackId: b, key: "db:sql", host: "127.0.0.1", port }, bind) + .pipe(Effect.flip); + + expect(failure.message).toContain(String(port)); + expect(failure.message).toContain(projectA); + expect(failure.message).toContain("supabase stack stop"); + expect(failure.message).not.toContain("destroy"); + if (!(failure instanceof PortError)) return yield* Effect.die("expected a PortError"); + expect(failure.conflict?.holder).toMatchObject({ stackId: a }); + }), + ).pipe(withRegistry), +); + +it.live( + "a pinned reservation left behind by a dead owner is released when the next owner stops", + () => + Effect.scoped( + Effect.gen(function* () { + const { ports, registry, stateRoot, a, b } = yield* twoProjects; + const port = pinnedPort(); + const bind = (_host: string, bound: number) => Effect.succeed(bound); + yield* registry.reserve(stateRoot, a, "db:sql", port); + + yield* Effect.scoped( + ports.acquire({ stackId: a, key: "db:sql", host: "127.0.0.1", port }, bind), + ); + + expect(yield* registry.find(stateRoot, a, "db:sql")).toBeUndefined(); + expect( + (yield* ports.acquire({ stackId: b, key: "db:sql", host: "127.0.0.1", port }, bind)).port, + ).toBe(port); + }), + ).pipe(withRegistry), +); + +it.live( + "a pinned port held by an owner that died without closing is free for another project", + () => + Effect.scoped( + Effect.gen(function* () { + const { ports, registry, stateRoot, a, b } = yield* twoProjects; + const port = pinnedPort(); + yield* registry.reserve(stateRoot, a, "db:sql", port); + + const started = yield* ports.acquire( + { stackId: b, key: "db:sql", host: "127.0.0.1", port }, + (_host, bound) => Effect.succeed(bound), + ); + + expect(started.port).toBe(port); + expect(yield* registry.find(stateRoot, b, "db:sql")).toBe(port); + }), + ).pipe(withRegistry), +); + +it.live("a pinned port that fails to bind is not left reserved", () => + Effect.scoped( + Effect.gen(function* () { + const { ports, registry, stateRoot, a } = yield* twoProjects; + const port = pinnedPort(); + + yield* ports + .acquire({ stackId: a, key: "db:sql", host: "127.0.0.1", port }, () => + Effect.fail(new PortError({ key: "db:sql", message: "bind failed" })), + ) + .pipe(Effect.flip); + + expect(yield* registry.find(stateRoot, a, "db:sql")).toBeUndefined(); + }), + ).pipe(withRegistry), +); + +it.live("a pinned port inside the native backend range names the range and the endpoint", () => + Effect.scoped( + Effect.gen(function* () { + const { ports, a } = yield* twoProjects; + + const failure = yield* ports + .acquire({ stackId: a, key: "db:sql", host: "127.0.0.1", port: 15432 }, (_host, bound) => + Effect.succeed(bound), + ) + .pipe(Effect.flip); + + expect(failure.message).toContain("db:sql"); + expect(failure.message).toContain("native service backends"); + }), + ).pipe(withRegistry), +); diff --git a/packages/stack/src/Ports.ts b/packages/stack/src/Ports.ts index a232141fd7..d23ea3ced6 100644 --- a/packages/stack/src/Ports.ts +++ b/packages/stack/src/Ports.ts @@ -8,10 +8,12 @@ import { Option, Path, Predicate, + Schema, Scope, } from "effect"; import * as Net from "node:net"; import type * as StackNamespace from "./StackNamespace.ts"; +import * as Lease from "./namespace/Lease.ts"; import * as PortReservations from "./namespace/PortReservations.ts"; import type { Holder } from "./namespace/PortReservations.ts"; @@ -125,7 +127,7 @@ export const probeVacant = if (yield* loopbackVacant(platform)(port)) return; return yield* new PortError({ key, - message: `Port ${port} for ${key} at ${host}:${port} is already in use`, + message: `Port ${port} for ${key} at ${host}:${port} is in use by another process; free it or configure a different port for ${key}`, conflict: { port, endpoint: key, holder: "foreign" }, }); }); @@ -253,16 +255,20 @@ const resolveRequest = ( return Effect.fail( new PortError({ key: request.key, - message: `Public port ${requested} is inside ${nativePortBase}-${portBase - 1}, which is reserved for native service ports; choose a port outside that range`, + message: `Public port ${requested} for ${request.key} is inside ${nativePortBase}-${portBase - 1}, which is reserved for native service backends; configure a port outside that range`, }), ); return Effect.succeed(requested); }; +const RegisteredProject = Schema.fromJsonString( + Schema.Struct({ identity: Schema.Struct({ projectRoot: Schema.String }) }), +); + /** * One stack's public ports, reserved through the per-user registry before any physical listener - * exists: a reservation is committed first, then the listener is created, so a stopped stack's - * ports stay reserved across every other stack's starts and across state roots. + * exists: a reservation is committed first, then the listener is created. An auto port stays + * reserved while its stack is stopped; a pinned port is reserved only while its listener is open. */ export const makePorts = (state: StackNamespace.Interface) => Effect.gen(function* () { @@ -272,8 +278,15 @@ export const makePorts = (state: StackNamespace.Interface) => // The registry's state_root is the realpath, so a symlinked or relative root still matches // the identity another process derives from the same stack. const stateRoot = yield* fs.realPath(state.root); - - const describe = (holder: Holder) => `stack ${holder.stackId} at ${holder.stateRoot}`; + const services = yield* Effect.context(); + + /** The holder's project directory, read from its own registration; undefined when unreadable. */ + const projectOf = (holder: Holder) => + fs.readFileString(path.join(holder.stateRoot, holder.stackId, "state.json")).pipe( + Effect.flatMap(Schema.decodeEffect(RegisteredProject)), + Effect.map((saved) => saved.identity.projectRoot), + Effect.orElseSucceed(() => undefined), + ); /** Only a confirmed `ENOENT` on the holder's own registration makes its reservation stale. */ const isGone = (holder: Holder) => @@ -286,20 +299,31 @@ export const makePorts = (state: StackNamespace.Interface) => Effect.orElseSucceed(() => false), ); + /** Whether a process holds the holder stack's owner lease; an unreadable lease counts as held. */ + const isLeased = (holder: Holder) => + Lease.make({ root: holder.stateRoot, isRegistered: () => Effect.succeed(true) }).pipe( + Effect.flatMap((lease) => lease.leased(holder.stackId)), + Effect.provideContext(services), + Effect.orElseSucceed(() => true), + ); + /** - * Reserves `port` for `(stackId, key)`, running the lazy-reclamation check against a live - * conflict's own registration before giving up on it. Resolves to the still-live holder on - * conflict, or `undefined` once the row is ours. Untraced: an auto scan calls this per - * candidate port, and the enclosing `Ports.acquire` span already records the attempt count. + * Reserves `port` for `(stackId, key)`, running the lazy-reclamation check against a + * conflict's own registration before giving up on it. A pinned request also reclaims a + * holder with no live owner, since the owner that died never closed its listener. Resolves + * to the still-live holder on conflict, or `undefined` once the row is ours. Untraced: an + * auto scan calls this per candidate port, and the enclosing `Ports.acquire` span already + * records the attempt count. */ const reserveCandidate = Effect.fnUntraced(function* ( stackId: string, key: string, port: number, + pinned: boolean, ) { const holder = yield* portReservations.reserve(stateRoot, stackId, key, port); if (holder === undefined) return undefined; - if (!(yield* isGone(holder))) return holder; + if (!(yield* isGone(holder)) && !(pinned && !(yield* isLeased(holder)))) return holder; const reclaimed = yield* portReservations.reclaim(port, holder, { stateRoot, stackId, @@ -308,10 +332,21 @@ export const makePorts = (state: StackNamespace.Interface) => return reclaimed ? undefined : holder; }); - const holderMessage = (key: string, port: number, stackId: string, holder: Holder) => - holder.stackId === stackId - ? `Public port ${port} for ${key} is claimed by another listener of this stack` - : `Public port ${port} for ${key} is claimed by ${describe(holder)}`; + const holderMessage = Effect.fnUntraced(function* ( + key: string, + port: number, + stackId: string, + holder: Holder, + ) { + if (holder.stackId === stackId) + return `Public port ${port} for ${key} is claimed by another listener of this stack`; + const project = yield* projectOf(holder); + const where = + project === undefined + ? `stack ${holder.stackId}` + : `the stack of project ${project} (stack ${holder.stackId})`; + return `Public port ${port} for ${key} is in use by ${where}; run \`supabase stack stop\` in that project, or configure a different port for ${key}`; + }); const acquire = Effect.fn("Ports.acquire")(function* ( request: PortRequest, @@ -350,11 +385,38 @@ export const makePorts = (state: StackNamespace.Interface) => ), ); - /** One bind attempt at `port`, in its own forked scope so a failure closes the listener; a success holds the scope open instead. */ + /** + * One bind attempt at `port` in its own forked scope; a failure closes the scope, a + * success keeps it open. A pinned request's row lives exactly as long as that scope, so a + * stopped stack holds none; an auto assignment's row outlives it and is kept across stops. + */ const attemptBind = (port: number) => Effect.uninterruptibleMask((restore) => Effect.gen(function* () { + if (owned === undefined) { + const holder = yield* reserveCandidate( + request.stackId, + request.key, + port, + request.port !== "auto", + ); + if (holder !== undefined) + return Exit.fail( + new PortError({ + key: request.key, + message: yield* holderMessage(request.key, port, request.stackId, holder), + conflict: { port, endpoint: request.key, holder }, + }), + ); + } const scope = yield* Scope.fork(owner, "sequential"); + if (request.port !== "auto") + yield* Scope.addFinalizer( + scope, + portReservations + .release(stateRoot, request.stackId, request.key) + .pipe(Effect.orDie), + ); return yield* restore( bindListener(port).pipe(Effect.provideService(Scope.Scope, scope)), ).pipe( @@ -367,15 +429,6 @@ export const makePorts = (state: StackNamespace.Interface) => ); if (requested !== "auto") { - if (owned === undefined) { - const holder = yield* reserveCandidate(request.stackId, request.key, requested); - if (holder !== undefined) - return yield* new PortError({ - key: request.key, - message: holderMessage(request.key, requested, request.stackId, holder), - conflict: { port: requested, endpoint: request.key, holder }, - }); - } const result = yield* attemptBind(requested); if (Exit.isSuccess(result)) return result.value; const error = Cause.findErrorOption(result.cause); @@ -386,8 +439,6 @@ export const makePorts = (state: StackNamespace.Interface) => !(error.value instanceof PortError) ) return yield* Effect.failCause(result.cause); - // Owning this port already, or reserving it exactly, is sticky: a bind failure never - // reassigns it, and the reservation survives so the next attempt retries this port. return yield* error.value; } @@ -400,7 +451,7 @@ export const makePorts = (state: StackNamespace.Interface) => const attemptCandidate = (port: number) => Effect.uninterruptibleMask((restore) => Effect.gen(function* () { - const holder = yield* reserveCandidate(request.stackId, request.key, port); + const holder = yield* reserveCandidate(request.stackId, request.key, port, false); if (holder !== undefined) return undefined; const scope = yield* Scope.fork(owner, "sequential"); return yield* restore( From c674218c4c83f4ff844ebee2c23a60f08df5f539 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 14:38:01 +0200 Subject: [PATCH 130/142] docs(stack): describe lost ownership, pinned port reservations and failing destroy --- .../stack/destroy/SIDE_EFFECTS.md | 8 +-- .../experimental/stack/start/SIDE_EFFECTS.md | 13 ++-- .../experimental/stack/stop/SIDE_EFFECTS.md | 9 +-- packages/stack/ARCHITECTURE.md | 61 ++++++++----------- packages/stack/README.md | 4 +- 5 files changed, 43 insertions(+), 52 deletions(-) diff --git a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md index 78d2a0a996..0727127019 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md @@ -73,10 +73,10 @@ override) only once cleanup is confirmed complete; a destroy that fails keeps the stack's reservations, and its ports remain unavailable to other stacks until a later destroy succeeds. A reservation whose owning stack's state was deleted without going through destroy (for example a removed -or unmounted state root) is usually released by that stack's own still-running -owner within its next registration check (30 seconds by default), which also -stops its workloads; the lazy reclaim by whichever stack next needs that port -remains the backstop when no owner is left alive to do so. +or unmounted state root) is released lazily by whichever stack next needs that +port. That stack's still-running owner notices the lost registration within its +next check (30 seconds by default), stops its workloads and exits without +deleting data or releasing reservations. ## Output, exit codes and telemetry diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index ed59883c87..92d9c2cca5 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -45,11 +45,14 @@ PostgREST runs with `PGRST_LOG_LEVEL=info`, so every request line, query string persisted and shipped to Analytics. Public ports are reserved in one SQLite registry per OS user at `/.supabase/ports.sqlite`, -which `$SUPABASE_HOME` does not affect and which has no override. A stopped stack's ports stay -reserved across every other stack's starts, including in a different state root, and starting it -again reuses the same ports. A saved or explicitly requested port already held by another stack, or -occupied by a process outside the registry, fails start naming the conflict instead of picking a -different port; only automatic allocation tries another candidate. On macOS a process can still win +which `$SUPABASE_HOME` does not affect and which has no override. A stopped stack's automatic +ports stay reserved across every other stack's starts, including in a different state root, and +starting it again reuses the same ports. A configured port is reserved only while its listener is +open, so a stopped stack never blocks another project's configured port; a configured port takes +over a reservation whose stack has no running owner (after a crash or reboot), and that stack gets a +new automatic port on its next start. A saved or configured port held by another running stack, or +occupied by a process outside the registry, fails start naming the port and, for a stack, its +project, instead of picking a different port; only automatic allocation tries another candidate. On macOS a process can still win a narrow race against the pre-bind probe before a brand-new listener exists. Native backend ports (not publicly exposed) are reserved from 10000–19999, disjoint from the public auto range and below the OS ephemeral range on every supported platform; this range is not diff --git a/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md index 79580887b4..36eb5d7241 100644 --- a/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md @@ -40,10 +40,11 @@ Reads saved state under `/stacks//` and the current project/Git identity when no explicit ID is supplied. Shared routing and settings may read project configuration and the selected profile. Communicates only with local owner endpoints; no hosted API requests. Shutdown updates owned -runtime resources but preserves persistent instance definitions, data and port -reservations in the per-user registry at `/.supabase/ports.sqlite` -(unaffected by `SUPABASE_HOME`); killing the owner process preserves them the -same way. No caller-owned upload files are removed. +runtime resources but preserves persistent instance definitions, data and automatic +port reservations in the per-user registry at `/.supabase/ports.sqlite` +(unaffected by `SUPABASE_HOME`); configured ports are released with their listeners. +Killing the owner process preserves every reservation, but another stack may take over +a configured port while no owner runs. No caller-owned upload files are removed. ## Exit codes and telemetry diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index ca53d57fc4..503bd5d89e 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -358,7 +358,7 @@ flowchart TB Stopped -->|explicit start| Start["Orchestrator starts selected services
Reuses saved public ports"] ``` -Sleep retains the public listener needed to wake. Whole-stack stop closes listeners while preserving the registry's reservations; the next start rebinds the same ports. Destroying an individual instance deletes only the registry rows it owns, not unrelated services' shared listeners or rows; a stack-wide destroy deletes its rows only after the registration is removed, so a failed destroy keeps them and a failed release leaves them for lazy reclamation. +Sleep retains the public listener needed to wake. Whole-stack stop closes listeners and releases pinned reservations while preserving automatic ones; the next start rebinds the same ports. Destroying an individual instance deletes only the registry rows it owns, not unrelated services' shared listeners or rows; a stack-wide destroy deletes its rows only after the registration is removed, so a failed destroy keeps them and a failed release leaves them for lazy reclamation. Composition validation permits lazy activation only for instances with a configured public wake endpoint. A route-less prerequisite, such as pg-meta without its own public endpoint, is eager; it is not implicitly armed through a dependent. This avoids a second wake-permission mechanism. Internal sleep is available only for lazy instances with a supported public wake route and an enabled idle policy. The default composition gives every lazy service one, Functions included; the database is eager. Functions inspector access remains possible while health is starting; ordinary application traffic waits for healthy. @@ -491,24 +491,11 @@ During Starting, acquire the exclusive stack lease, load the instance definition **Orphan sweep.** Stack-labelled containers and session stacks exist only while their lease is held. After readiness, each owner visits every other stack in its state root in the background, with a bounded time per stack. It skips stacks whose lease is held. For a free lease it takes that lease for the duration of the visit, publishing a sweeper record in `owner.json` so clients wait for the visit instead of mistaking it for a starting owner, removes containers labelled with the stack and its data root, and destroys the stack through the owner's own destroy path when its lifetime is `session`. A container-runtime owner also visits stacks that have no registration but left containers labelled with this state root on its pinned engine, such as when their directory was deleted, under the same lease and unregistered check, and removes only those containers; the stack's data in the shared database volume stays. `findDeleted` selects such a stack by its full id, so a client can destroy those containers on demand. Filtering on the data-root label keeps other state roots untouched. Creating a stack whose identity belongs to a dead session stack reclaims that stack the same way first. The sweep also removes the empty `/` and `data/` directories a destroy leaves behind, once they are over ten minutes old and hold nothing else. -**Abandoned registration.** The orphan sweep above covers a dead owner whose registration survives; -the opposite case is a live owner whose registration disappears out from under it (the state root -is deleted, moved or unmounted). Every owner polls its own `//state.json` on an -interval (30 seconds by default) and treats only a confirmed ENOENT as gone; any other read error, -such as a permission failure or a transient I/O error on an unmounted root, keeps the owner -running and logs a warning instead. On a confirmed-gone registration the owner treats its -ownership as ended: after any stop already running settles, it runs the ordinary destroy once -(ordered stop, instance data removal, label sweep, port release) and exits. The destroy is best -effort and its failure is logged, not retried. Registration and state root need not exist for it -to complete: confirming the absent registration skips the registry lock, so the label sweep still -runs when the whole state root is gone. A transient engine failure during that single cleanup -leaves labelled containers behind; the next startup, stop or destroy of a stack with the same id -and data root sweeps them by label. Deletion, a move and an unmount are treated identically, -because a stack whose root is unreachable cannot operate regardless of which of the three caused it. - -**Engine targets.** A container stack pins the engine target it first resolves, so every command it runs reaches the same engine. Docker pins `DOCKER_HOST` as `--host`, else the current context. Podman pins `CONTAINER_HOST` as `--url`, else `CONTAINER_CONNECTION` as `--connection`; with neither set, a bare `podman info` decides whether the engine is local or remote. A remote engine (macOS `podman machine`) pins the default system connection by name, and a local one adds no flags. On rootless Podman, containers that run as the caller's uid with writable borrowed bind mounts get `--userns=keep-id` so files stay owned by the caller. Containers reach host listeners through the engine's host alias, `host.docker.internal` or `host.containers.internal`. - -**Unreachable engine.** An owner of a container stack fails startup with a `runtime-unavailable` reason when its first container sweep finds the engine CLI missing or its daemon or service not listening; permission, TLS, authentication and timeout failures are ordinary startup errors. `destroy` then proceeds without an owner: it only confirms that no owner holds the stack's lease, leaves the registration, port reservations and host data untouched, and returns `{ runtimeCleanup: "skipped", engine }`; run it again once the engine is reachable. `stop` without a live owner takes the same lease and reclaims the registered stack's leftovers (the orphan-sweep path: containers, plus destruction of a session stack) before succeeding; it fails if another process holds the lease or the cleanup fails, and so needs the engine for a container stack. +**Lost ownership.** The orphan sweep above covers a dead owner whose registration survives; the opposite case is a live owner whose registration disappears out from under it (the stack directory or state root is deleted, moved or unmounted, possibly followed by a new start of the same stack). Every owner polls on an interval (30 seconds by default) and treats its ownership as ended when `//state.json` is confirmed missing (ENOENT) or `owner.lock` at its path is no longer the file its lease locked. Any other read error keeps the owner running and logs a warning. When ownership ends, the owner stops what it launched once any stop already running settles, closes its listeners and exits. It does not destroy instance data, sweep labels, release port reservations or write the registry, and it retracts `owner.json` only while the lock file at its path is still its own, so a successor owner is never disturbed. Leftovers of a stack whose directory is gone are reclaimed by the existing paths: the next container owner's sweep of unregistered labelled stacks, lazy reclamation of reservations whose registration is gone, and `destroy --stack-id`. Docker database data in the shared volume of such a stack is not reclaimed. + +**Engine targets.** A container stack's owner pins the engine target it resolves at startup, so every command it runs reaches the same engine. The target is not saved with the stack: an owner started after the Docker context or `DOCKER_HOST` changed resolves the new target, and containers left on the previous engine are not reached from there; switch back to clean them up. Docker pins `DOCKER_HOST` as `--host`, else the current context. Podman pins `CONTAINER_HOST` as `--url`, else `CONTAINER_CONNECTION` as `--connection`; with neither set, a bare `podman info` decides whether the engine is local or remote. A remote engine (macOS `podman machine`) pins the default system connection by name, and a local one adds no flags. On rootless Podman, containers that run as the caller's uid with writable borrowed bind mounts get `--userns=keep-id` so files stay owned by the caller. Containers reach host listeners through the engine's host alias, `host.docker.internal` or `host.containers.internal`. + +**Unreachable engine.** An owner of a container stack fails startup with a `runtime-unavailable` reason when its first container sweep finds the engine CLI missing or its daemon or service not listening; permission, TLS, authentication and timeout failures are ordinary startup errors. `destroy` then fails with that reason, leaving the registration, port reservations and host data untouched; run it again once the engine is reachable. `stop` without a live owner takes the same lease and reclaims the registered stack's leftovers (the orphan-sweep path: containers, plus destruction of a session stack) before succeeding; it fails if another process holds the lease or the cleanup fails, and so needs the engine for a container stack. During Serving, keep the owner alive independently of callers. Sleeping instances still need its public listeners. This is process lifetime management, not automatic service restart or continuous reconciliation. @@ -599,13 +586,13 @@ Do not persist health, runtime lifecycle projections, transition progress, comma Some foundations are not optional, but they do not need to dominate the service model: -| Foundation | Keep | -| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------- | -| Identity | Stack and instance IDs; deterministic names and paths for their resources | -| Ownership | One detached StackHost and exclusive lease; client handles do not own service lifetime | -| Ports | Persist public port assignments across stop/start; reuse the same ports, including initially automatic assignments; never silently relocate | -| Data | Stop retains data; destroy removes only proven owned data | -| Snapshots | DB-specific keyed save, compatible empty-target restore, complete entry publication and credential reconciliation | +| Foundation | Keep | +| ---------- | ----------------------------------------------------------------------------------------------------------------------------- | +| Identity | Stack and instance IDs; deterministic names and paths for their resources | +| Ownership | One detached StackHost and exclusive lease; client handles do not own service lifetime | +| Ports | Persist automatic port assignments across stop/start; a pinned port is reserved only while listening; never silently relocate | +| Data | Stop retains data; destroy removes only proven owned data | +| Snapshots | DB-specific keyed save, compatible empty-target restore, complete entry publication and credential reconciliation | ### Port allocation and endpoint ownership @@ -627,7 +614,7 @@ A shared listener is a networking resource, not a capability or service group. C A lazy database already has a bound SQL proxy listener and a concrete connection URL; lazy REST already has its path on the bound shared API listener. Their backend processes may still be absent. The first request launches the needed backends, waits for readiness and forwards through the already published endpoint. Backend address allocation can wait until that launch; public port assignment cannot. If a required listener cannot bind, startup returns an error with partial outcomes rather than reporting successful startup with unusable URLs. -A host-wide allocation registry coordinates claims across parallel stacks, including stopped instances and stacks. Before first listener activation, obtain an assignment and bind it. Automatic allocation excludes managed claims and tries to bind an available candidate. Reserve the assignment in the registry before advertising it and retain that socket; do not probe then release it. Later activations bind the saved port. Fixed requests obey the same ownership and binding checks. +A host-wide allocation registry coordinates claims across parallel stacks. Before first listener activation, obtain an assignment and bind it. Automatic allocation excludes managed claims and tries to bind an available candidate. Reserve the assignment in the registry before advertising it and retain that socket; do not probe then release it. Later activations bind the saved port. An automatic assignment stays reserved while its stack is stopped. A configured (pinned) port is reserved only while its listener is open, so a stopped stack blocks no other project; the endpoint's address always comes from the configured number, never from the registry. A pinned port inside the native backend range is rejected. A conflict names the port and, for a registry holder, its project and the commands that free it. ```mermaid flowchart LR @@ -638,18 +625,18 @@ flowchart LR Runtime["Native or container runtime"] -.-> Backend ``` -| Operation | Dedicated instance listener | Route on a shared listener | -| ---------------------- | --------------------------------------------------------------- | ---------------------------------------------------------- | -| Start | Bind saved port and connect backend | Enable this route and bind the saved shared port if needed | -| Restart | Retain assignment and update target | Retain shared assignment and update this route's target | -| Sleep | Keep listener for wake | Keep route armed for wake | -| Stop | Close listener; retain claim | Disable this route; leave other routes alone | -| Instance destroy | Remove endpoint and release its claim | Remove this route; retain the stack-owned listener claim | -| Namespace stop/destroy | Close all listeners; stop retains claims, destroy releases them | Same rule | +| Operation | Dedicated instance listener | Route on a shared listener | +| ---------------------- | ------------------------------------------------------------------------- | ---------------------------------------------------------- | +| Start | Bind saved port and connect backend | Enable this route and bind the saved shared port if needed | +| Restart | Retain assignment and update target | Retain shared assignment and update this route's target | +| Sleep | Keep listener for wake | Keep route armed for wake | +| Stop | Close listener; retain an automatic claim | Disable this route; leave other routes alone | +| Instance destroy | Remove endpoint and release its claim | Remove this route; retain the stack-owned listener claim | +| Namespace stop/destroy | Close all listeners; stop retains automatic claims, destroy releases them | Same rule | -A shared listener stays bound while any route is running or armed. When no route needs it, its socket may close while its stack-owned claim remains. Removing the final route does not release that claim; namespace destruction or explicit listener reconfiguration does. Sleeping routes still need the listener. +A shared listener stays bound while any route is running or armed. When no route needs it, its socket closes; an automatic claim remains, a pinned one is released with the socket. Removing the final route does not release an automatic claim; namespace destruction or explicit listener reconfiguration does. Sleeping routes still need the listener. -**Public port stickiness is a retained contract.** Sleep/wake, stop/start, restart and host reopening preserve every established assignment, including the shared API port. Other managed stacks cannot take retained claims. An unrelated process may occupy a saved port while its socket is closed; the next activation reports a conflict and preserves the assignment, never silently relocating it. A destroyed and recreated instance has a new identity and receives a new dedicated allocation. +**Public port stickiness is a retained contract for automatic assignments.** Sleep/wake, stop/start, restart and host reopening preserve every established automatic assignment, including the shared API port. Automatic allocation never takes another stack's retained claim. A pinned port is the configuration's own number, so it needs no retention; its registry row exists only while the listener is open, and a pinned request reclaims a row whose stack has no live owner (a free owner lease), such as after a crash or reboot. If that row was a stopped stack's automatic assignment, that stack receives a new automatic port on its next start. An unrelated process may occupy a saved port while its socket is closed; the next activation reports a conflict and preserves the assignment, never silently relocating it. A destroyed and recreated instance has a new identity and receives a new dedicated allocation. Standalone and composed instances use the same allocator. Two shadow databases have separate IDs, data and dedicated listeners. Composition membership does not change listener ownership. The runtime reports the backend address; the proxy updates its target without altering the public assignment. diff --git a/packages/stack/README.md b/packages/stack/README.md index a24d70c131..69e35fb1e9 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -69,7 +69,7 @@ On Linux, native Functions project files must be outside `/tmp`: Edge Runtime us `open({ id, stateRoot, cacheRoot })` reconnects to a saved stack. The package stores the stack document at `//state.json` and service data at `//data/`. `discover({ stateRoot })` lists saved definitions separately from live-owner availability. It skips each entry that cannot be read or decoded and reports it to `onInvalidState(id, error)`; only a failure to read `stateRoot` itself fails discovery. Port allocation skips the same entries. Offline definitions are not live lifecycle observations. -`find({ stateRoot, projectRoot, name })` derives the stack ID with the same identity rules as `create` and reads only that stack; `find({ stateRoot, id })` reads a known ID. It returns the saved definition with the live owner's endpoint, if any, or nothing when no such stack is saved. Unlike `discover`, an unreadable state document fails the call instead of being skipped. The Effect entrypoint's `StackId` schema validates an ID before lookup. +`find({ stateRoot, projectRoot, name })` derives the stack ID with the same identity rules as `create` and reads only that stack; `find({ stateRoot, id })` reads a known ID. It returns the saved definition with the live owner's endpoint, if any, or nothing when no such stack is saved. Unlike `discover`, an unreadable state document fails the call, naming the stack's directory, instead of being skipped. The Effect entrypoint's `StackId` schema validates an ID before lookup. Pass `startOwner: true` to `open` when live status and other owner-backed operations are needed; this starts only the detached owner and does not start services. The owner writes its output to `//owner.log`, which each owner start truncates; owner start and connection failures name that file. A client drives only an owner of its own release; other operations fail and ask you to stop or destroy the stack, which work across releases. @@ -79,7 +79,7 @@ Pass `startOwner: true` to `open` when live status and other owner-backed operat `create({ ..., startOwner: true })` starts a detached stack's owner immediately and lets it register the stack under its lease, as a session stack always does. If the owner fails to start or the launch is interrupted before it reports ready, the owner removes that registration and `create` fails with the launch error, so a failed launch leaves no stack behind. -`destroy` normally returns `{ runtimeCleanup: "complete" }`. When no owner is running and a new owner cannot start because the stack's container engine (Docker or Podman) cannot be reached, `destroy` confirms that no owner holds the stack's lease and leaves the registration, port reservations and host data untouched, returning `{ runtimeCleanup: "skipped", engine }`. When an owner does run but a labelled container cannot be removed (for example one on a different daemon) or an instance's data cannot be removed, `destroy` likewise keeps the stack registered and fails, listing what remains. Either way, retry `destroy` once the engine is reachable; there is no `cleanupCommands` field. +`destroy` succeeds once the stack and everything it owns are removed. When no owner is running and a new owner cannot start because the stack's container engine (Docker or Podman) cannot be reached, `destroy` fails with reason `runtime-unavailable` and leaves the registration, port reservations and host data untouched. When an owner does run but a labelled container cannot be removed (for example one on a different daemon) or an instance's data cannot be removed, `destroy` likewise keeps the stack registered and fails, listing what remains. Either way, retry `destroy` once the failure is resolved; there is no `cleanupCommands` field. The stack owns database, Functions bootstrap, and command-job directories below its data directory. Storage uploads remain at the caller-supplied Storage `filePath` and are preserved when the stack is destroyed; the caller owns that directory. Host metadata remains under `stateRoot`; native database data uses host files. Docker database data normally uses a managed volume, while existing host data is retained through the host-backed fallback; Podman always keeps database data in a host-backed directory. A host marker records the selected Docker storage and detects a missing or mismatched volume; deleting that volume loses the associated database data. Native snapshot entries live below `cacheRoot`. Docker snapshots share the managed data volume in a separate namespace derived from `cacheRoot`, so they survive source destruction and can use filesystem cloning. A Docker cache hit requires the same daemon, `stateRoot`, and `cacheRoot`. There is no portable tar snapshot API. From ad75865f4d554ef87dc5125bed912d2c2e806d7f Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 14:42:51 +0200 Subject: [PATCH 131/142] fix(cli): name the config setting when a stack start hits a port conflict --- .../experimental/stack/start/start.handler.ts | 33 +++++++++++++++++-- .../start/start.native.integration.test.ts | 6 ++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/apps/cli/src/commands/experimental/stack/start/start.handler.ts b/apps/cli/src/commands/experimental/stack/start/start.handler.ts index bfc83abf92..4af819f86d 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.handler.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.handler.ts @@ -94,18 +94,45 @@ const validateExclusions = (exclusions: ReadonlyArray) => { ); }; +/** + * Names the config setting behind a contested public port. A conflicting endpoint key ends in its + * endpoint name (the shared API listener is keyed `api`); only a name that maps to one setting + * across the requested services gets a suggestion. + */ +const portConflictSuggestion = ( + conflict: StackError["conflict"], + services: ReadonlyArray<{ readonly service: string }>, +): string | undefined => { + if (conflict === undefined) return undefined; + const settings = + conflict.endpoint === "api" + ? [stackEndpointSetting("rest", "http")] + : services.map(({ service }) => + stackEndpointSetting( + service, + conflict.endpoint.slice(conflict.endpoint.indexOf(":") + 1), + ), + ); + const [setting, ...rest] = settings.filter((candidate) => candidate !== undefined); + return setting === undefined || rest.some(({ envVar }) => envVar !== setting.envVar) + ? undefined + : `Set \`${setting.configPath}\` in supabase/config.toml (or ${setting.envVar}) to a free port.`; +}; + const stackError = ( - cause: { readonly message: string } & Partial>, - members: ReadonlyArray<{ readonly id: string; readonly service: string }> = [], + cause: { readonly message: string } & Partial>, + members: ReadonlyArray<{ readonly id?: string; readonly service: string }> = [], ) => { const detail = failedOutcomesDetail(cause, (id) => { const service = members.find((member) => member.id === id)?.service; return service === undefined ? id : `${service} (${id})`; }); + const suggestion = portConflictSuggestion(cause.conflict, members); return new StackCommandStartError({ reason: "unknown", message: cause.message, ...(detail === undefined ? {} : { detail }), + ...(suggestion === undefined ? {} : { suggestion }), cause, }); }; @@ -785,7 +812,7 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags }) .pipe( Effect.tapError((error) => starting.fail(error.message)), - Effect.mapError(stackError), + Effect.mapError((error) => stackError(error, requested)), ); if (initialComposition) { const existingIds = new Set(existingServices.map(({ id }) => id)); diff --git a/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts index b7ffa80e82..21fdd22d8b 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts @@ -384,6 +384,12 @@ describe("experimental stack start native lifecycle", () => { `127\\.0\\.0\\.1:${occupiedPort}(: Cannot bind TCP listener| is in use by another process)`, ), ); + expect(Cause.findErrorOption(failedStart.cause)).toMatchObject( + Option.some({ + suggestion: + "Set `db.port` in supabase/config.toml (or SUPABASE_DB_PORT) to a free port.", + }), + ); const ownerPid = ownerPids[attempt]; expect(ownerPid).toBeDefined(); if (ownerPid === undefined) return yield* Effect.die("stack owner PID missing"); From 26fa409f6c45545be75651e57e5b3ecddd2f95b3 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 14:56:53 +0200 Subject: [PATCH 132/142] fix(cli): resolve the conflicting port setting by its configured port number --- .../experimental/stack/start/start.handler.ts | 42 ++++++------- .../start/start.native.integration.test.ts | 59 +++++++++++++++++++ 2 files changed, 80 insertions(+), 21 deletions(-) diff --git a/apps/cli/src/commands/experimental/stack/start/start.handler.ts b/apps/cli/src/commands/experimental/stack/start/start.handler.ts index 4af819f86d..45a3dfd5e4 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.handler.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.handler.ts @@ -94,27 +94,29 @@ const validateExclusions = (exclusions: ReadonlyArray) => { ); }; +const isRecord = (value: unknown): value is Readonly> => + typeof value === "object" && value !== null; + /** - * Names the config setting behind a contested public port. A conflicting endpoint key ends in its - * endpoint name (the shared API listener is keyed `api`); only a name that maps to one setting - * across the requested services gets a suggestion. + * Names the config setting behind a contested public port: the one configured endpoint pinned to + * that port number. An automatic port, or a port shared by several settings, gets no suggestion. */ const portConflictSuggestion = ( conflict: StackError["conflict"], - services: ReadonlyArray<{ readonly service: string }>, + requested: ReadonlyArray<{ readonly service: string; readonly endpoints?: unknown }>, ): string | undefined => { if (conflict === undefined) return undefined; - const settings = - conflict.endpoint === "api" - ? [stackEndpointSetting("rest", "http")] - : services.map(({ service }) => - stackEndpointSetting( - service, - conflict.endpoint.slice(conflict.endpoint.indexOf(":") + 1), - ), - ); - const [setting, ...rest] = settings.filter((candidate) => candidate !== undefined); - return setting === undefined || rest.some(({ envVar }) => envVar !== setting.envVar) + const settings = new Map(); + for (const { service, endpoints } of requested) { + if (!isRecord(endpoints)) continue; + for (const [name, intent] of Object.entries(endpoints)) { + const setting = stackEndpointSetting(service, name); + if (setting !== undefined && isRecord(intent) && intent.port === conflict.port) + settings.set(setting.envVar, setting); + } + } + const [setting, ...rest] = settings.values(); + return setting === undefined || rest.length > 0 ? undefined : `Set \`${setting.configPath}\` in supabase/config.toml (or ${setting.envVar}) to a free port.`; }; @@ -122,12 +124,13 @@ const portConflictSuggestion = ( const stackError = ( cause: { readonly message: string } & Partial>, members: ReadonlyArray<{ readonly id?: string; readonly service: string }> = [], + requested: ReadonlyArray<{ readonly service: string; readonly endpoints?: unknown }> = [], ) => { const detail = failedOutcomesDetail(cause, (id) => { const service = members.find((member) => member.id === id)?.service; return service === undefined ? id : `${service} (${id})`; }); - const suggestion = portConflictSuggestion(cause.conflict, members); + const suggestion = portConflictSuggestion(cause.conflict, requested); return new StackCommandStartError({ reason: "unknown", message: cause.message, @@ -210,9 +213,6 @@ const studioNeedsRest = (requestedServices: ReadonlyArray<{ readonly service: st requestedServices.some(({ service }) => service === "studio") && !requestedServices.some(({ service }) => service === "rest"); -const isRecord = (value: unknown): value is Readonly> => - typeof value === "object" && value !== null; - const endpointPortLabel = (endpoints: unknown, name: string): string => { const intent = isRecord(endpoints) ? endpoints[name] : undefined; const port = isRecord(intent) ? intent.port : undefined; @@ -812,7 +812,7 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags }) .pipe( Effect.tapError((error) => starting.fail(error.message)), - Effect.mapError((error) => stackError(error, requested)), + Effect.mapError((error) => stackError(error, requested, requested)), ); if (initialComposition) { const existingIds = new Set(existingServices.map(({ id }) => id)); @@ -936,7 +936,7 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags } yield* stack.composition.start.pipe( Effect.tapError((error) => starting.fail(error.message)), - Effect.mapError((error) => stackError(error, members)), + Effect.mapError((error) => stackError(error, members, requested)), ); yield* Effect.forEach(preparation, (fiber) => Fiber.join(fiber)); yield* Ref.set(startupComplete, true); diff --git a/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts index 21fdd22d8b..f540a2edee 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts @@ -325,6 +325,65 @@ describe("experimental stack start native lifecycle", () => { { timeout: 180_000 }, ); + it.live( + "names the pinned Studio port setting when another process holds that port", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-native-studio-" }); + yield* fs.makeDirectory(path.join(root, "supabase"), { recursive: true }); + const server = net.createServer(); + const ownedServer = yield* Effect.acquireRelease( + Effect.callback((resume) => { + const onError = (cause: Error) => resume(Effect.fail(cause)); + server.once("error", onError); + server.listen(0, "127.0.0.1", () => { + server.removeListener("error", onError); + resume(Effect.succeed(server)); + }); + }), + (listeningServer) => + Effect.callback((resume) => { + listeningServer.close(() => resume(Effect.void)); + }), + ); + const address = ownedServer.address(); + if (address === null || typeof address === "string") + return yield* Effect.die("Unable to reserve a native test port"); + yield* fs.writeFileString( + path.join(root, "supabase", "config.toml"), + projectConfig.replace( + "[studio]\nenabled = false", + `[studio]\nenabled = true\nport = ${address.port}`, + ), + ); + const fixture = makeLayers(root); + yield* Effect.scoped( + Effect.gen(function* () { + const api = yield* StackApi; + yield* Effect.addFinalizer(() => + destroyTestStacks(api, path.join(root, "stacks"), path.join(root, "cache")), + ); + const failedStart = yield* Effect.scoped( + Effect.exit( + stackStart(flags(excluded.filter((name) => name !== "studio" && name !== "rest"))), + ), + ); + expect(Exit.isFailure(failedStart)).toBe(true); + if (!Exit.isFailure(failedStart)) return; + expect(Cause.findErrorOption(failedStart.cause)).toMatchObject( + Option.some({ + suggestion: + "Set `studio.port` in supabase/config.toml (or SUPABASE_STUDIO_PORT) to a free port.", + }), + ); + }), + ).pipe(Effect.provide(fixture.layer)); + }).pipe(Effect.provide(BunServices.layer)), + { timeout: 180_000 }, + ); + it.live( "stops owners after bind and pre-compose config failures across retries", () => From 87cd62a237b5b83cd2f90a285dd2cd66beb3e6e7 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 15:01:06 +0200 Subject: [PATCH 133/142] fix(stack): check the data root before a service destroy and skip a symlinked staging dir --- packages/stack/src/Owner.integration.test.ts | 39 +++++++++++++++++++ packages/stack/src/Owner.ts | 7 +++- .../src/namespace/Paths.integration.test.ts | 18 +++++++++ packages/stack/src/namespace/Paths.ts | 17 ++++---- .../stack/src/preparation/ArtifactStore.ts | 18 +++++---- .../preparation/artifacts.integration.test.ts | 29 ++++++++++++++ 6 files changed, 111 insertions(+), 17 deletions(-) diff --git a/packages/stack/src/Owner.integration.test.ts b/packages/stack/src/Owner.integration.test.ts index ca72b84be8..87c73cf9d0 100644 --- a/packages/stack/src/Owner.integration.test.ts +++ b/packages/stack/src/Owner.integration.test.ts @@ -893,6 +893,45 @@ it.live("destroying a stack leaves its data root empty after an owner was killed ), ); +it.live("refuses to destroy one service after its data directory became a symlink", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-service-symlink-" }); + const stack = initial(`owner-service-symlink-${randomUUID().slice(0, 8)}`); + const state = yield* stateFor(`${root}/state`); + yield* state.save(stack); + const data = `${root}/state/${stack.id}/data`; + yield* fs.makeDirectory(data, { recursive: true }); + const owner = yield* ownerFor({ saved: stack, state, root: data, cacheRoot }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + const service = yield* owner.rpc.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" }, smtp: { port: "auto" }, pop3: { port: "auto" } }, + }); + const outside = `${root}/outside`; + yield* fs.makeDirectory(`${outside}/${service.id}`, { recursive: true }); + yield* fs.writeFileString(`${outside}/${service.id}/precious.txt`, "keep"); + yield* fs.remove(data, { recursive: true }); + yield* fs.symlink(outside, data); + + const failure = yield* owner.rpc.destroyService({ id: service.id }).pipe(Effect.flip); + + expect(failure.message).toContain("symlink"); + expect(yield* fs.readFileString(`${outside}/${service.id}/precious.txt`)).toBe("keep"); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), +); + it.live("refuses to destroy a stack whose data directory is a symlink and keeps its target", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index 95f413d60c..220ed3e1d9 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -773,7 +773,12 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { rpcError("restartService"), ), destroyService: ({ id }) => - definitionChange(orchestrator.destroy(id)).pipe(rpcError("destroyService")), + definitionChange( + confirmStackDataRoot(options.state.root, stackId).pipe( + Effect.provideContext(services), + Effect.andThen(orchestrator.destroy(id)), + ), + ).pipe(rpcError("destroyService")), prepareService: ({ id }) => orchestrator.get(id).pipe( Effect.flatMap((entry) => diff --git a/packages/stack/src/namespace/Paths.integration.test.ts b/packages/stack/src/namespace/Paths.integration.test.ts index d134d9c84b..5482b207a1 100644 --- a/packages/stack/src/namespace/Paths.integration.test.ts +++ b/packages/stack/src/namespace/Paths.integration.test.ts @@ -242,4 +242,22 @@ describe("the stack data root", () => { }), ), ); + + it.live("is refused at launch when it is a dangling symlink", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-data-root-dangling-" }); + yield* fs.makeDirectory(path.join(root, "state", "stack"), { recursive: true }); + yield* fs.symlink(path.join(root, "missing"), path.join(root, "state", "stack", "data")); + + const failure = yield* resolveStackDataRoot(path.join(root, "state"), "stack").pipe( + Effect.flip, + ); + + expect(failure.message).toContain("symlink"); + }), + ), + ); }); diff --git a/packages/stack/src/namespace/Paths.ts b/packages/stack/src/namespace/Paths.ts index 948e197119..d7824b03d5 100644 --- a/packages/stack/src/namespace/Paths.ts +++ b/packages/stack/src/namespace/Paths.ts @@ -163,9 +163,9 @@ export const destroyOwnedRoot = Effect.fn("Namespace.Paths.destroyOwnedRoot")( ); /** - * Refuses a stack's `data` directory when it is a symlink or resolves outside the stack - * directory, so a destroy never empties a target the stack does not own. A missing directory - * passes: only launch creates it. + * Refuses a stack's `data` directory when it is a symlink (dangling or not), not a directory, or + * resolves outside the stack directory, so nothing is created in or removed from a target the stack + * does not own. A missing directory passes: only launch creates it. */ export const confirmStackDataRoot = Effect.fn("Namespace.Paths.confirmStackDataRoot")(function* ( stateRoot: string, @@ -175,10 +175,11 @@ export const confirmStackDataRoot = Effect.fn("Namespace.Paths.confirmStackDataR const path = yield* Path.Path; const stackRoot = path.join(stateRoot, stackId); const dataRoot = path.join(stackRoot, "data"); - const present = yield* fs - .exists(dataRoot) - .pipe(Effect.mapError((cause) => namespaceError("destroy", cause))); - if (present) yield* confirmRealOwnedDirectory(fs, path, dataRoot, stackRoot, namespaceError); + const info = yield* lstatPath(dataRoot).pipe( + Effect.mapError((cause) => namespaceError("destroy", cause)), + ); + if (info !== undefined) + yield* confirmRealOwnedDirectory(fs, path, dataRoot, stackRoot, namespaceError); }); /** @@ -194,7 +195,7 @@ export const resolveStackDataRoot = Effect.fn("Namespace.Paths.resolveStackDataR const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const dataRoot = path.join(stateRoot, stackId, "data"); - yield* fs.makeDirectory(dataRoot, { recursive: true }); yield* confirmStackDataRoot(stateRoot, stackId); + yield* fs.makeDirectory(dataRoot, { recursive: true }); return yield* fs.realPath(dataRoot); }); diff --git a/packages/stack/src/preparation/ArtifactStore.ts b/packages/stack/src/preparation/ArtifactStore.ts index 4702632c74..5ed94c3beb 100644 --- a/packages/stack/src/preparation/ArtifactStore.ts +++ b/packages/stack/src/preparation/ArtifactStore.ts @@ -590,6 +590,14 @@ const reapStaleStaging = ( ), ); +/** A symlink is never a cache entry: the cache neither walks nor writes through one. */ +const isSymlink = (fs: FileSystem.FileSystem, target: string): Effect.Effect => + // readLink succeeds only on a symlink. + fs.readLink(target).pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + ); + interface CacheWalk { readonly generations: ReadonlyArray<{ readonly keyRoot: string; readonly digest: string }>; readonly stagingRoots: ReadonlyArray; @@ -613,14 +621,7 @@ const walkCache = ( const generations: Array = []; const stagingRoots: Array = []; for (const name of names) { - // readLink succeeds only on a symlink; a link is never a cache entry, so it is left alone. - if ( - yield* fs.readLink(path.join(root, name)).pipe( - Effect.as(true), - Effect.orElseSucceed(() => false), - ) - ) - continue; + if (yield* isSymlink(fs, path.join(root, name))) continue; if (name === STAGING_DIR_NAME) { stagingRoots.push(path.join(root, name)); continue; @@ -664,6 +665,7 @@ const retireGeneration = Effect.fn("ArtifactStore.retireGeneration")(function* ( const now = yield* Clock.currentTimeMillis; if (mtime === undefined || now - mtime.getTime() <= RETIREMENT_AGE_MILLIS) return undefined; + if (yield* isSymlink(fs, stagingRoot)) return undefined; yield* fs.makeDirectory(stagingRoot, { recursive: true, mode: 0o700 }); const token = yield* crypto.randomUUIDv4; const slotLockPath = stagingLockFor(path, stagingRoot, token); diff --git a/packages/stack/src/preparation/artifacts.integration.test.ts b/packages/stack/src/preparation/artifacts.integration.test.ts index e1313ce8ad..94ef0482ac 100644 --- a/packages/stack/src/preparation/artifacts.integration.test.ts +++ b/packages/stack/src/preparation/artifacts.integration.test.ts @@ -744,6 +744,35 @@ describe("lock-guarded staging", () => { 20_000, ); + it.live.skipIf(process.platform === "win32")( + "retires nothing through a symlinked staging root under an aged unpinned generation", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-retire-link-", + }); + const outside = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-retire-outside-", + }); + const key = "database/postgres-retire-link"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const prepared = yield* store.prepare({ ...request, key }); + yield* fs.writeFileString(`${outside}/precious`, "keep"); + yield* fs.remove(`${root}/${key}/.staging`, { recursive: true, force: true }); + yield* fs.symlink(outside, `${root}/${key}/.staging`); + yield* ageLockFile(fs, prepared.lockPath); + + yield* triggerSweep(root); + + expect(yield* fs.readDirectory(outside)).toEqual(["precious"]); + expect(yield* fs.exists(prepared.path)).toBe(true); + }), + ), + 20_000, + ); + it.live( "leaves a staging lock file that has no directory yet for the preparer that is about to take it", () => From cdf3db2bd0d400f0b7091ff351d7992c546b605b Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 15:01:06 +0200 Subject: [PATCH 134/142] docs(stack): qualify reservation release and destroy coverage after lost ownership --- .../commands/experimental/stack/destroy/SIDE_EFFECTS.md | 7 ++++--- packages/stack/ARCHITECTURE.md | 4 ++-- packages/stack/README.md | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md index 0727127019..266acaa3fb 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md @@ -70,13 +70,14 @@ routing/settings can read project configuration and profiles. Destruction releases this stack's rows in the per-user port registry at `/.supabase/ports.sqlite` (independent of `SUPABASE_HOME`, with no override) only once cleanup is confirmed complete; a destroy that fails -keeps the stack's reservations, and its ports remain unavailable -to other stacks until a later destroy succeeds. A reservation whose owning +keeps the stack's automatic reservations, and those ports remain unavailable +to other stacks until a later destroy succeeds; configured ports are released +with their listeners. A reservation whose owning stack's state was deleted without going through destroy (for example a removed or unmounted state root) is released lazily by whichever stack next needs that port. That stack's still-running owner notices the lost registration within its next check (30 seconds by default), stops its workloads and exits without -deleting data or releasing reservations. +deleting data; its configured ports are released with their listeners. ## Output, exit codes and telemetry diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 503bd5d89e..6bcbb730de 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -358,7 +358,7 @@ flowchart TB Stopped -->|explicit start| Start["Orchestrator starts selected services
Reuses saved public ports"] ``` -Sleep retains the public listener needed to wake. Whole-stack stop closes listeners and releases pinned reservations while preserving automatic ones; the next start rebinds the same ports. Destroying an individual instance deletes only the registry rows it owns, not unrelated services' shared listeners or rows; a stack-wide destroy deletes its rows only after the registration is removed, so a failed destroy keeps them and a failed release leaves them for lazy reclamation. +Sleep retains the public listener needed to wake. Whole-stack stop closes listeners and releases pinned reservations while preserving automatic ones; the next start rebinds the same ports. Destroying an individual instance deletes only the registry rows it owns, not unrelated services' shared listeners or rows; a stack-wide destroy deletes its rows only after the registration is removed, so a failed destroy keeps the automatic ones (a configured port's row is released when its listener closes) and a failed release leaves them for lazy reclamation. Composition validation permits lazy activation only for instances with a configured public wake endpoint. A route-less prerequisite, such as pg-meta without its own public endpoint, is eager; it is not implicitly armed through a dependent. This avoids a second wake-permission mechanism. Internal sleep is available only for lazy instances with a supported public wake route and an enabled idle policy. The default composition gives every lazy service one, Functions included; the database is eager. Functions inspector access remains possible while health is starting; ordinary application traffic waits for healthy. @@ -491,7 +491,7 @@ During Starting, acquire the exclusive stack lease, load the instance definition **Orphan sweep.** Stack-labelled containers and session stacks exist only while their lease is held. After readiness, each owner visits every other stack in its state root in the background, with a bounded time per stack. It skips stacks whose lease is held. For a free lease it takes that lease for the duration of the visit, publishing a sweeper record in `owner.json` so clients wait for the visit instead of mistaking it for a starting owner, removes containers labelled with the stack and its data root, and destroys the stack through the owner's own destroy path when its lifetime is `session`. A container-runtime owner also visits stacks that have no registration but left containers labelled with this state root on its pinned engine, such as when their directory was deleted, under the same lease and unregistered check, and removes only those containers; the stack's data in the shared database volume stays. `findDeleted` selects such a stack by its full id, so a client can destroy those containers on demand. Filtering on the data-root label keeps other state roots untouched. Creating a stack whose identity belongs to a dead session stack reclaims that stack the same way first. The sweep also removes the empty `/` and `data/` directories a destroy leaves behind, once they are over ten minutes old and hold nothing else. -**Lost ownership.** The orphan sweep above covers a dead owner whose registration survives; the opposite case is a live owner whose registration disappears out from under it (the stack directory or state root is deleted, moved or unmounted, possibly followed by a new start of the same stack). Every owner polls on an interval (30 seconds by default) and treats its ownership as ended when `//state.json` is confirmed missing (ENOENT) or `owner.lock` at its path is no longer the file its lease locked. Any other read error keeps the owner running and logs a warning. When ownership ends, the owner stops what it launched once any stop already running settles, closes its listeners and exits. It does not destroy instance data, sweep labels, release port reservations or write the registry, and it retracts `owner.json` only while the lock file at its path is still its own, so a successor owner is never disturbed. Leftovers of a stack whose directory is gone are reclaimed by the existing paths: the next container owner's sweep of unregistered labelled stacks, lazy reclamation of reservations whose registration is gone, and `destroy --stack-id`. Docker database data in the shared volume of such a stack is not reclaimed. +**Lost ownership.** The orphan sweep above covers a dead owner whose registration survives; the opposite case is a live owner whose registration disappears out from under it (the stack directory or state root is deleted, moved or unmounted, possibly followed by a new start of the same stack). Every owner polls on an interval (30 seconds by default) and treats its ownership as ended when `//state.json` is confirmed missing (ENOENT) or `owner.lock` at its path is no longer the file its lease locked. Any other read error keeps the owner running and logs a warning. When ownership ends, the owner stops what it launched once any stop already running settles, closes its listeners and exits. It does not destroy instance data, sweep labels or write the registry, and it retracts `owner.json` only while the lock file at its path is still its own, so a successor owner's lease is never disturbed. Closing the listeners releases the reservations of configured ports; automatic reservations are kept. A stop already running when the directory is deleted and the same stack restarted can still sweep the successor's containers by label. Leftovers of a stack whose directory is gone are reclaimed by the existing paths: the next container owner's sweep of unregistered labelled stacks, lazy reclamation of reservations whose registration is gone, and `destroy --stack-id`. Docker database data in the shared volume of such a stack is not reclaimed. **Engine targets.** A container stack's owner pins the engine target it resolves at startup, so every command it runs reaches the same engine. The target is not saved with the stack: an owner started after the Docker context or `DOCKER_HOST` changed resolves the new target, and containers left on the previous engine are not reached from there; switch back to clean them up. Docker pins `DOCKER_HOST` as `--host`, else the current context. Podman pins `CONTAINER_HOST` as `--url`, else `CONTAINER_CONNECTION` as `--connection`; with neither set, a bare `podman info` decides whether the engine is local or remote. A remote engine (macOS `podman machine`) pins the default system connection by name, and a local one adds no flags. On rootless Podman, containers that run as the caller's uid with writable borrowed bind mounts get `--userns=keep-id` so files stay owned by the caller. Containers reach host listeners through the engine's host alias, `host.docker.internal` or `host.containers.internal`. diff --git a/packages/stack/README.md b/packages/stack/README.md index 69e35fb1e9..dd4601a3fe 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -79,7 +79,7 @@ Pass `startOwner: true` to `open` when live status and other owner-backed operat `create({ ..., startOwner: true })` starts a detached stack's owner immediately and lets it register the stack under its lease, as a session stack always does. If the owner fails to start or the launch is interrupted before it reports ready, the owner removes that registration and `create` fails with the launch error, so a failed launch leaves no stack behind. -`destroy` succeeds once the stack and everything it owns are removed. When no owner is running and a new owner cannot start because the stack's container engine (Docker or Podman) cannot be reached, `destroy` fails with reason `runtime-unavailable` and leaves the registration, port reservations and host data untouched. When an owner does run but a labelled container cannot be removed (for example one on a different daemon) or an instance's data cannot be removed, `destroy` likewise keeps the stack registered and fails, listing what remains. Either way, retry `destroy` once the failure is resolved; there is no `cleanupCommands` field. +`destroy` succeeds once the stack and everything it owns are removed, except containers left on a container engine other than the one currently selected, which are cleaned only after switching back (see Engine targets in `ARCHITECTURE.md`). When no owner is running and a new owner cannot start because the stack's container engine (Docker or Podman) cannot be reached, `destroy` fails with reason `runtime-unavailable` and leaves the registration, port reservations and host data untouched. When an owner does run but a labelled container cannot be removed (for example one on a different daemon) or an instance's data cannot be removed, `destroy` likewise keeps the stack registered and fails, listing what remains. Either way, retry `destroy` once the failure is resolved; there is no `cleanupCommands` field. The stack owns database, Functions bootstrap, and command-job directories below its data directory. Storage uploads remain at the caller-supplied Storage `filePath` and are preserved when the stack is destroyed; the caller owns that directory. Host metadata remains under `stateRoot`; native database data uses host files. Docker database data normally uses a managed volume, while existing host data is retained through the host-backed fallback; Podman always keeps database data in a host-backed directory. A host marker records the selected Docker storage and detects a missing or mismatched volume; deleting that volume loses the associated database data. Native snapshot entries live below `cacheRoot`. Docker snapshots share the managed data volume in a separate namespace derived from `cacheRoot`, so they survive source destruction and can use filesystem cloning. A Docker cache hit requires the same daemon, `stateRoot`, and `cacheRoot`. There is no portable tar snapshot API. @@ -106,7 +106,7 @@ field names the port, the endpoint, and either the live holder (`{ stackId, stat `"foreign"` for a process outside the registry. A fixed or previously saved port is never silently reassigned; only automatic allocation tries another candidate. A reservation is released when deleting an individual service releases the endpoints only it owned, or when a stack-wide `destroy` -has removed the registration. A failed stack-wide `destroy` keeps every reservation; a failed +has removed the registration. A failed stack-wide `destroy` keeps automatic reservations (a configured port's reservation is released when its listener closes); a failed individual-service destroy may already have released some of that service's endpoints, and the service can stay unavailable until the destroy is retried. Stopping a stack or killing its owner leaves reservations in place. A reservation whose owning stack's `state.json` is confirmed gone (`ENOENT`, for example From f7b0da381c371dcb242f3e3fdc0ed1213d057852 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 15:09:27 +0200 Subject: [PATCH 135/142] test(stack): wait for a swept session stack's directory to disappear --- packages/stack/src/Sweep.integration.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/stack/src/Sweep.integration.test.ts b/packages/stack/src/Sweep.integration.test.ts index 8bf25aebde..9fbaef2dcf 100644 --- a/packages/stack/src/Sweep.integration.test.ts +++ b/packages/stack/src/Sweep.integration.test.ts @@ -178,7 +178,7 @@ it.live.skipIf(process.platform === "win32")( [ awaitContainerRemoved(orphan, since), awaitContainerRemoved(deleted, since), - awaitRemoval(path.join(rootA, sessionId), "state.json"), + awaitRemoval(rootA, sessionId), awaitRemoval(path.join(rootA, deadId), "owner.json"), ], { concurrency: "unbounded" }, From 454bf1748c5fbfd4aa385dda30084f2695858e5b Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 15:20:33 +0200 Subject: [PATCH 136/142] test(stack): drop the socket sweep case for a symlinked data root, which launch now refuses --- ...er.native-socket-sweep.integration.test.ts | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/packages/stack/src/Owner.native-socket-sweep.integration.test.ts b/packages/stack/src/Owner.native-socket-sweep.integration.test.ts index 0cd0035c5f..753aef61cf 100644 --- a/packages/stack/src/Owner.native-socket-sweep.integration.test.ts +++ b/packages/stack/src/Owner.native-socket-sweep.integration.test.ts @@ -206,29 +206,6 @@ it.live( ).pipe(Effect.provide(platform)), ); -it.live( - "an orphan reclaim finds the socket directory of a stack whose data root is a symlink", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const arrangement = yield* arrange("socket-symlink", "session"); - const relocated = `${arrangement.root}/relocated-data`; - yield* fs.makeDirectory(relocated); - yield* fs.symlink(relocated, arrangement.dataRoot); - // Launch hashes the data root's real path, so a leftover is named from it. - const leftover = yield* leaveSocketDirectory( - { ...arrangement, dataRoot: relocated }, - "database", - ); - - expect(yield* reclaim(arrangement)).toBe(true); - - expect(yield* fs.exists(leftover)).toBe(false); - }), - ).pipe(Effect.provide(platform)), -); - it.live("removes only the socket directory of the stack being cleaned up", () => Effect.scoped( Effect.gen(function* () { From d74142fec37b989bdcc0f988814556b0219a3f89 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 15:48:02 +0200 Subject: [PATCH 137/142] test(stack): share one host gateway probe across container adapter tests --- .../src/runtime/Container.integration.test.ts | 116 +++++++++++------- 1 file changed, 72 insertions(+), 44 deletions(-) diff --git a/packages/stack/src/runtime/Container.integration.test.ts b/packages/stack/src/runtime/Container.integration.test.ts index 1863cc41ca..28f841047b 100644 --- a/packages/stack/src/runtime/Container.integration.test.ts +++ b/packages/stack/src/runtime/Container.integration.test.ts @@ -17,6 +17,7 @@ import { Redacted, Ref, Schema, + Scope, Sink, Stream, } from "effect"; @@ -50,6 +51,11 @@ const image = await Effect.gen(function* () { // empty argv prefix leaves every command exactly as it was before pinning existed. const containerTarget: EngineTarget = { ...engineTarget, argv: [] }; const ipv4 = /^(?:\d{1,3}\.){3}\d{1,3}$/u; +// One probe for every test that does not assert probe behaviour, so engine load stays bounded. +const sharedHostGateway = await makeHostGateway.pipe( + Effect.provideService(Scope.Scope, await Effect.runPromise(Scope.make())), + Effect.runPromise, +); const stoppableIdleScript = "process.on('SIGTERM', () => process.exit(0)); setInterval(() => {}, 1000)"; @@ -58,7 +64,7 @@ class ContainerTestError extends Data.TaggedError("ContainerTestError")<{ readonly cause?: unknown; }> {} -describe("container process adapter", () => { +describe("container process adapter", { timeout: 120_000 }, () => { it.live("recognizes a cached pinned image without pulling", () => Effect.gen(function* () { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; @@ -113,6 +119,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const process = yield* runtime.launch({ @@ -150,6 +157,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const process = yield* runtime.launchCommand({ @@ -565,6 +573,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const process = yield* runtime.launch({ @@ -592,6 +601,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const process = yield* runtime.launchCommand({ @@ -618,6 +628,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const crypto = yield* Crypto.Crypto; @@ -651,6 +662,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const launch = (instanceId: string, marker: string) => @@ -690,6 +702,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const process = yield* runtime.launch({ @@ -719,6 +732,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const process = yield* runtime.launch({ @@ -749,6 +763,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const result = yield* runtime @@ -780,6 +795,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const process = yield* runtime.launch({ @@ -811,6 +827,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }).pipe( Effect.provideService( ChildProcessSpawner.ChildProcessSpawner, @@ -852,6 +869,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }).pipe( Effect.provideService( ChildProcessSpawner.ChildProcessSpawner, @@ -885,6 +903,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const process = yield* runtime.launch({ @@ -920,6 +939,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const process = yield* runtime.launch({ @@ -960,6 +980,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const process = yield* runtime.launch({ @@ -1005,6 +1026,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const process = yield* runtime.launch({ @@ -1036,49 +1058,47 @@ describe("container process adapter", () => { }).pipe(Effect.provide(NodeServices.layer)), ); - it.live( - "waits for a real removal observed as removing", - () => - Effect.gen(function* () { - const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; - const crypto = yield* Crypto.Crypto; - const token = yield* crypto.randomUUIDv4; - const instanceId = `pending-remove-${token}`; - const pendingProbes = yield* Ref.make(2); - const lostResult = yield* Ref.make(false); - const spawner = makePendingRemoveSpawner(delegate, pendingProbes, lostResult); - yield* Effect.ensuring( - Effect.gen(function* () { - const result = yield* Effect.scoped( - Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ - target: containerTarget, - root: ".", - }); - yield* runtime.prepare(image); - const process = yield* runtime.launch({ - image, - stackId: "n".repeat(64), - instanceId, - env: {}, - args: ["-e", stoppableIdleScript], - }); - yield* process.stop; - yield* process.remove; - }), - ).pipe( - Effect.exit, - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - ); - expect(yield* Ref.get(lostResult)).toBe(true); - expect(yield* Ref.get(pendingProbes)).toBe(0); - expect(Exit.isSuccess(result)).toBe(true); - expect(yield* idsByInstance(instanceId)).toHaveLength(0); - }), - removeByInstance(instanceId).pipe(Effect.orDie), - ); - }).pipe(Effect.provide(NodeServices.layer)), - { timeout: 120_000 }, + it.live("waits for a real removal observed as removing", () => + Effect.gen(function* () { + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const crypto = yield* Crypto.Crypto; + const token = yield* crypto.randomUUIDv4; + const instanceId = `pending-remove-${token}`; + const pendingProbes = yield* Ref.make(2); + const lostResult = yield* Ref.make(false); + const spawner = makePendingRemoveSpawner(delegate, pendingProbes, lostResult); + yield* Effect.ensuring( + Effect.gen(function* () { + const result = yield* Effect.scoped( + Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); + yield* runtime.prepare(image); + const process = yield* runtime.launch({ + image, + stackId: "n".repeat(64), + instanceId, + env: {}, + args: ["-e", stoppableIdleScript], + }); + yield* process.stop; + yield* process.remove; + }), + ).pipe( + Effect.exit, + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + expect(yield* Ref.get(lostResult)).toBe(true); + expect(yield* Ref.get(pendingProbes)).toBe(0); + expect(Exit.isSuccess(result)).toBe(true); + expect(yield* idsByInstance(instanceId)).toHaveLength(0); + }), + removeByInstance(instanceId).pipe(Effect.orDie), + ); + }).pipe(Effect.provide(NodeServices.layer)), ); it.live("retains the remove failure when removing never reaches absence", () => @@ -1097,6 +1117,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const process = yield* runtime.launch({ @@ -1147,6 +1168,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const process = yield* runtime.launch({ @@ -1194,6 +1216,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const process = yield* runtime.launch({ @@ -1250,6 +1273,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const result = yield* runtime @@ -1294,6 +1318,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); return yield* runtime @@ -1337,6 +1362,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }); yield* runtime.prepare(image); const launch = yield* runtime @@ -1369,6 +1395,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }).pipe(Effect.provide(engine.layer)); const launch = yield* Effect.scoped( runtime.launch({ image, stackId: "i".repeat(64), instanceId: "hung-create", env: {} }), @@ -1394,6 +1421,7 @@ describe("container process adapter", () => { const runtime = yield* makeContainerRuntime({ target: containerTarget, root: ".", + hostGateway: sharedHostGateway, }).pipe(Effect.provide(engine.layer)); const launch = yield* Effect.scoped( runtime.launch({ image, stackId: "i".repeat(64), instanceId: "interrupted", env: {} }), From 068238832c08537d2c15a8da0c59429791f94894 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 16:43:19 +0200 Subject: [PATCH 138/142] test(stack): align port reservation e2e with configured-port takeover and the native range message --- packages/stack/README.md | 8 +-- .../stack/src/port-reservation.e2e.test.ts | 54 ++++++++----------- 2 files changed, 26 insertions(+), 36 deletions(-) diff --git a/packages/stack/README.md b/packages/stack/README.md index 27a7c97e6d..7b448e1584 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -100,8 +100,8 @@ Every public port is reserved, before any listener binds it, in one SQLite regis user database (`getent passwd ` on Linux, `dscacheutil -q user -a uid ` on macOS), never `$HOME` or `SUPABASE_HOME`, and with no override; a process cannot steer the registry's location by setting its own environment. The registry is the only saved port assignment (the stack's `state.json` holds none) and the only authority on which stack owns a port across -every state root on the machine, so a stopped stack keeps its ports across other -stacks' starts, and a restart reuses the same ports or fails with a `StackError` whose `conflict` +every state root on the machine, so a stopped stack keeps its automatic ports across other +stacks' automatic allocation, and a restart reuses the same ports or fails with a `StackError` whose `conflict` field names the port, the endpoint, and either the live holder (`{ stackId, stateRoot }`) or `"foreign"` for a process outside the registry. A fixed or previously saved port is never silently reassigned; only automatic allocation tries another candidate. A reservation is released when @@ -109,7 +109,9 @@ deleting an individual service releases the endpoints only it owned, or when a s has removed the registration. A failed stack-wide `destroy` keeps automatic reservations (a configured port's reservation is released when its listener closes); a failed individual-service destroy may already have released some of that service's endpoints, and the service can stay unavailable until the destroy is retried. Stopping a stack or killing its -owner leaves reservations in place. A reservation whose owning stack's `state.json` is confirmed gone (`ENOENT`, for example +owner leaves automatic reservations in place, but a configured port claimed by another stack takes +over a reservation whose stack has no running owner; that stack's next start allocates a new +automatic port. A reservation whose owning stack's `state.json` is confirmed gone (`ENOENT`, for example after deleting its state root) is reclaimed lazily by the next stack that needs its port. Known limitations: on macOS, BSD, and Windows, where the kernel allows overlapping binds, a loopback diff --git a/packages/stack/src/port-reservation.e2e.test.ts b/packages/stack/src/port-reservation.e2e.test.ts index 804ab5eaf7..1d55cd9476 100644 --- a/packages/stack/src/port-reservation.e2e.test.ts +++ b/packages/stack/src/port-reservation.e2e.test.ts @@ -11,9 +11,9 @@ import { testEngine } from "../tests/test-engine.ts"; import type { ContainerEngine } from "./runtime/Container.ts"; /** - * The per-user port reservation registry: a stopped stack keeps its public ports across - * every other stack's starts and across state roots, a restart recovers the same ports or fails - * with a structured `PortConflict`, and destroy (but never stop or a kill) releases them. + * The per-user port reservation registry: a stopped stack keeps its automatic ports across state + * roots unless another stack's configured port takes one over, a restart recovers the same ports + * or fails with a structured `PortConflict`, and destroy (but never stop or a kill) releases them. */ const servicesLayer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); @@ -102,7 +102,7 @@ const withFakeHome = (home: string, use: Effect.Effect) => ); it.live( - "a stopped stack's reserved port survives another stack's start in a different state root; a fixed claim on it is a structured PortConflict naming the holder, and the stack restarts on the same port", + "a configured port takes over a stopped stack's automatic port in another state root, and the stopped stack restarts on a new automatic port", () => run( Effect.gen(function* () { @@ -113,38 +113,26 @@ it.live( yield* mailA.start; yield* mailA.ready; const port = portOf(yield* mailA.status); - - // A is stopped, not destroyed: its registration, and the public port it was given, persist. - // `Stack.stop` already confirms the owner (and so its listener) has exited. yield* stackA.stop; const { stack: stackB } = yield* makeStack(fs, cacheRoot); const mailB = yield* stackB.services.create(mailOn(port)); - const startB = yield* Effect.exit(mailB.start); - expect(Exit.isFailure(startB)).toBe(true); - expect(conflictHolderStackId(startB)).toBe(stackA.id); + yield* mailB.start; + yield* mailB.ready; + expect(portOf(yield* mailB.status)).toBe(port); - // A's restart recovers its own reserved port; B never legitimately held it. const reopenedA = yield* open({ id: stackA.id, stateRoot: stateRootA, cacheRoot }); const reopenedMailA = yield* reopenedA.services.get(mailA.id); yield* reopenedMailA.start; yield* reopenedMailA.ready; - expect(portOf(yield* reopenedMailA.status)).toBe(port); - - // Destroy releases the reservation: a third stack can now claim the exact same port. - yield* reopenedA.destroy; - const { stack: stackC } = yield* makeStack(fs, cacheRoot); - const mailC = yield* stackC.services.create(mailOn(port)); - yield* mailC.start; - yield* mailC.ready; - expect(portOf(yield* mailC.status)).toBe(port); + expect(portOf(yield* reopenedMailA.status)).not.toBe(port); }), ), { timeout: 120_000 }, ); it.live( - "a SIGKILLed owner's reservation survives: another stack's claim on its port is a conflict, the original restarts on the same port, and destroy then releases it", + "a SIGKILLed owner's automatic reservation survives, the stack restarts on the same port, and destroy then releases it", () => run( Effect.gen(function* () { @@ -163,12 +151,6 @@ it.live( // The owner's own exit already confirms the kernel has reclaimed its listeners. yield* waitForOwnerExit(pid, ownerExitProbe(fs)); - const { stack: stackB } = yield* makeStack(fs, cacheRoot); - const mailB = yield* stackB.services.create(mailOn(port)); - const startB = yield* Effect.exit(mailB.start); - expect(Exit.isFailure(startB)).toBe(true); - expect(conflictHolderStackId(startB)).toBe(stackA.id); - const reopenedA = yield* open({ id: stackA.id, stateRoot: stateRootA, cacheRoot }); const reopenedMailA = yield* reopenedA.services.get(mailA.id); yield* reopenedMailA.start; @@ -176,11 +158,17 @@ it.live( expect(portOf(yield* reopenedMailA.status)).toBe(port); yield* reopenedA.destroy; - const { stack: stackC } = yield* makeStack(fs, cacheRoot); - const mailC = yield* stackC.services.create(mailOn(port)); - yield* mailC.start; - yield* mailC.ready; - expect(portOf(yield* mailC.status)).toBe(port); + const portReservations = Context.get( + yield* Layer.build(PortReservations.layer), + PortReservations.Service, + ); + expect( + yield* portReservations.find( + yield* fs.realPath(stateRootA), + stackA.id, + `${mailA.id}:http`, + ), + ).toBeUndefined(); }), ), { timeout: 120_000 }, @@ -404,7 +392,7 @@ it.live( const failure = yield* Effect.flip(mail.start); expect(failure.message).toContain("10000-19999"); - expect(failure.message).toContain("choose a port outside that range"); + expect(failure.message).toContain("reserved for native service backends"); const reservationContext = yield* Layer.build(PortReservations.layer); const portReservations = Context.get(reservationContext, PortReservations.Service); expect( From 8f81ff399e342126038479033ca7f58322016973 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 17:48:25 +0200 Subject: [PATCH 139/142] fix(stack): run the attached database helper through the owner's pinned engine target --- packages/stack/src/runtime/Container.integration.test.ts | 5 ++++- packages/stack/src/storage/DockerDatabaseStorage.ts | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/stack/src/runtime/Container.integration.test.ts b/packages/stack/src/runtime/Container.integration.test.ts index 28f841047b..da4cb5f163 100644 --- a/packages/stack/src/runtime/Container.integration.test.ts +++ b/packages/stack/src/runtime/Container.integration.test.ts @@ -1523,7 +1523,10 @@ describe("container process adapter", { timeout: 120_000 }, () => { if (unpinned[0] === "version") return Effect.succeed(handle(0, "27.0.0|27.0.0")); if (unpinned[0] === "inspect") return Effect.succeed(handle(0, '{"Ports":{},"Status":"running","ExitCode":0}')); - if (unpinned[0] === "run") return Effect.succeed(handle(0, "abcdef012345")); + if (unpinned[0] === "run") + return Effect.succeed( + handle(0, unpinned.includes("-i") ? "supabase-helper-ready\n" : "abcdef012345"), + ); // Two call sites share "ps": the launched container's own `--rm` removal poll (filtered // by name, always confirmed gone) and the label-based orphan discovery below. if (unpinned[0] === "ps" && unpinned.some((arg) => arg.includes("com.supabase.stack="))) { diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index 8640ab08f3..fea5ee6e65 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -553,8 +553,9 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") const child = yield* options.spawner .spawn( ChildProcess.make( - options.runtime, + options.target.engine, [ + ...options.target.argv, "run", "--rm", "-i", From 9f31cf7fd09234c950e1db61691058eb8ca80b2b Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 18:08:51 +0200 Subject: [PATCH 140/142] fix(stack): refuse a stack directory that is a symlink or resolves outside the state root --- .../src/namespace/Paths.integration.test.ts | 23 +++++++++++++++++++ packages/stack/src/namespace/Paths.ts | 12 +++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/packages/stack/src/namespace/Paths.integration.test.ts b/packages/stack/src/namespace/Paths.integration.test.ts index 5482b207a1..b9fe0f9fea 100644 --- a/packages/stack/src/namespace/Paths.integration.test.ts +++ b/packages/stack/src/namespace/Paths.integration.test.ts @@ -243,6 +243,29 @@ describe("the stack data root", () => { ), ); + it.live("is refused when the stack directory is a symlink to an outside directory", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-dir-symlink-" }); + const outside = path.join(root, "outside"); + yield* fs.makeDirectory(path.join(outside, "data"), { recursive: true }); + const survivor = path.join(outside, "data", "keep"); + yield* fs.writeFileString(survivor, "do not remove me"); + yield* fs.makeDirectory(path.join(root, "state")); + yield* fs.symlink(outside, path.join(root, "state", "stack")); + + const failure = yield* resolveStackDataRoot(path.join(root, "state"), "stack").pipe( + Effect.flip, + ); + + expect(failure.message).toContain("symlink"); + expect(yield* fs.exists(survivor)).toBe(true); + }), + ), + ); + it.live("is refused at launch when it is a dangling symlink", () => run( Effect.gen(function* () { diff --git a/packages/stack/src/namespace/Paths.ts b/packages/stack/src/namespace/Paths.ts index d7824b03d5..35cfe439e0 100644 --- a/packages/stack/src/namespace/Paths.ts +++ b/packages/stack/src/namespace/Paths.ts @@ -163,9 +163,10 @@ export const destroyOwnedRoot = Effect.fn("Namespace.Paths.destroyOwnedRoot")( ); /** - * Refuses a stack's `data` directory when it is a symlink (dangling or not), not a directory, or - * resolves outside the stack directory, so nothing is created in or removed from a target the stack - * does not own. A missing directory passes: only launch creates it. + * Refuses a stack directory that is a symlink or resolves outside the state root, and a stack's + * `data` directory when it is a symlink (dangling or not), not a directory, or resolves outside the + * stack directory, so nothing is created in or removed from a target the stack does not own. A + * missing directory passes: only launch creates it. */ export const confirmStackDataRoot = Effect.fn("Namespace.Paths.confirmStackDataRoot")(function* ( stateRoot: string, @@ -175,6 +176,11 @@ export const confirmStackDataRoot = Effect.fn("Namespace.Paths.confirmStackDataR const path = yield* Path.Path; const stackRoot = path.join(stateRoot, stackId); const dataRoot = path.join(stackRoot, "data"); + const stackInfo = yield* lstatPath(stackRoot).pipe( + Effect.mapError((cause) => namespaceError("destroy", cause)), + ); + if (stackInfo !== undefined) + yield* confirmRealOwnedDirectory(fs, path, stackRoot, stateRoot, namespaceError); const info = yield* lstatPath(dataRoot).pipe( Effect.mapError((cause) => namespaceError("destroy", cause)), ); From e5939fe0a7bb1492b28f6e679904bae2bdc9ce4d Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 18:08:53 +0200 Subject: [PATCH 141/142] fix(stack): report a structured owner startup error without the owner log tail --- packages/stack/src/HostProcess.integration.test.ts | 5 +++-- packages/stack/src/HostProcess.ts | 6 +++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/stack/src/HostProcess.integration.test.ts b/packages/stack/src/HostProcess.integration.test.ts index 35fb1912f0..985f8e4f74 100644 --- a/packages/stack/src/HostProcess.integration.test.ts +++ b/packages/stack/src/HostProcess.integration.test.ts @@ -476,7 +476,7 @@ it.live("terminates a detached child when readiness is interrupted", () => ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); -it.live("keeps the owner log tail in a start failure after the owner removed its log", () => +it.live("reports a structured owner startup error without the owner log tail", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -490,7 +490,8 @@ it.live("keeps the owner log tail in a start failure after the owner removed its register: savedStack(root, "failing"), }).pipe(Effect.flip); expect(failure.message).toContain("owner startup failed"); - expect(failure.message).toContain("failing-owner-diagnostic"); + expect(failure.message).toContain("owner log:"); + expect(failure.message).not.toContain("failing-owner-diagnostic"); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); diff --git a/packages/stack/src/HostProcess.ts b/packages/stack/src/HostProcess.ts index e95a48b080..bf27544438 100644 --- a/packages/stack/src/HostProcess.ts +++ b/packages/stack/src/HostProcess.ts @@ -540,7 +540,11 @@ const spawnOwner = Effect.fn("HostProcess.spawnOwner")(function* ( } return yield* line.reason === "lease-held" || line.reason === "exists" ? error("startup", "Stack already exists; use open") - : failure(line.message, line.reason); + : error( + "startup", + `Stack owner failed to start: ${line.message} (owner log: ${log})`, + line.reason, + ); } if (line.endpoint.stackId !== options.stackId) return yield* failure( From ab4c0daea0d47f8fdab1c3c16a5637fbc25fc543 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 7 Oct 2026 18:08:54 +0200 Subject: [PATCH 142/142] fix(stack): let a configured port take over a reservation held only by a sweeper --- packages/stack/src/Ports.integration.test.ts | 26 ++++++++++++++++++++ packages/stack/src/Ports.ts | 17 +++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/packages/stack/src/Ports.integration.test.ts b/packages/stack/src/Ports.integration.test.ts index 411ba9f721..bd75ac0008 100644 --- a/packages/stack/src/Ports.integration.test.ts +++ b/packages/stack/src/Ports.integration.test.ts @@ -2,6 +2,7 @@ import { NodeFileSystem, NodeHttpClient, NodeServices } from "@effect/platform-n import { expect, it } from "@effect/vitest"; import { Context, + DateTime, Effect, Exit, FileSystem, @@ -599,6 +600,31 @@ it.live( ).pipe(withRegistry), ); +it.live("a configured port takes over a reservation whose stack is only held by a sweeper", () => + Effect.scoped( + Effect.gen(function* () { + const { state, ports, registry, stateRoot, a, b } = yield* twoProjects; + const port = pinnedPort(); + yield* registry.reserve(stateRoot, a, "db:sql", port); + const lease = yield* state.acquireLease(a); + yield* lease.publishHolder({ + role: "sweeper", + pid: process.pid, + startedAt: DateTime.formatIso(yield* DateTime.now), + }); + + const started = yield* ports.acquire( + { stackId: b, key: "db:sql", host: "127.0.0.1", port }, + (_host, bound) => Effect.succeed(bound), + ); + + expect(started.port).toBe(port); + expect(yield* registry.find(stateRoot, b, "db:sql")).toBe(port); + expect(yield* registry.find(stateRoot, a, "db:sql")).toBeUndefined(); + }), + ).pipe(withRegistry), +); + it.live("a pinned port that fails to bind is not left reserved", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/Ports.ts b/packages/stack/src/Ports.ts index d23ea3ced6..025ad27fe0 100644 --- a/packages/stack/src/Ports.ts +++ b/packages/stack/src/Ports.ts @@ -299,10 +299,23 @@ export const makePorts = (state: StackNamespace.Interface) => Effect.orElseSucceed(() => false), ); - /** Whether a process holds the holder stack's owner lease; an unreadable lease counts as held. */ + /** + * Whether a process holds the holder stack's owner lease; an unreadable lease counts as held. + * A sweeper holding it does not count, since a sweeper never uses the stack's public ports. + */ const isLeased = (holder: Holder) => Lease.make({ root: holder.stateRoot, isRegistered: () => Effect.succeed(true) }).pipe( - Effect.flatMap((lease) => lease.leased(holder.stackId)), + Effect.flatMap((lease) => + lease + .leased(holder.stackId) + .pipe( + Effect.flatMap((leased) => + leased + ? lease.readHolder(holder.stackId).pipe(Effect.map((h) => h?.role !== "sweeper")) + : Effect.succeed(false), + ), + ), + ), Effect.provideContext(services), Effect.orElseSucceed(() => true), );